diff --git a/.github/DEVELOPMENT.md b/.github/DEVELOPMENT.md index 793a2a1f375c..df79990a332b 100644 --- a/.github/DEVELOPMENT.md +++ b/.github/DEVELOPMENT.md @@ -1,28 +1,34 @@ # Development +In this document you can find information about developing Trino. + +* [Trino organization](#trino-organization) +* [Trino developer guide](#trino-developer-guide) +* [Code style](#code-style) +* [Additional IDE configuration](#additional-ide-configuration) +* [Building docs](#building-docs) +* [Building the Web UI](#building-the-web-ui) +* [Releases](#releases) + +## Trino organization + Learn about development for all Trino organization projects: * [Vision](https://trino.io/development/vision) * [Contribution process](https://trino.io/development/process#contribution-process) -* [Pull request and commit guidelines](https://trino.io/development/process#pull-request-and-commit-guidelines-) -* [Release note guidelines](https://trino.io/development/process#release-note-guidelines-) +* [Pull request and commit guidelines](https://trino.io/development/process#pull-request-and-commit-guidelines) +* [Release note guidelines](https://trino.io/development/process#release-note-guidelines) Further information in the [development section of the website](https://trino.io/development) includes different roles, like contributors, reviewers, and maintainers, related processes, and other aspects. +## Trino developer guide + See [the Trino developer guide](https://trino.io/docs/current/develop.html) for -information about the SPI, implementing connectors and other plugins plugins, +information about the SPI, implementing connectors and other plugins, the client protocol, writing tests and other lower level details. -More information about writing and building the documentation can be found in -the [docs module](../docs). - -* [Code style](#code-style) -* [Additional IDE configuration](#additional-ide-configuration) -* [Building the Web UI](#building-the-web-ui) -* [CI pipeline](#ci-pipeline) - ## Code Style We recommend you use IntelliJ as your IDE. The code style template for the @@ -214,6 +220,11 @@ with `@Language`: - Local variables which otherwise would not be properly recognized by IDE for language injection. +## Building docs + +Information about writing and building the documentation can be found in +the [docs module](../docs). + ## Building the Web UI The Trino Web UI is composed of several React components and is written in JSX diff --git a/.github/actions/compile-commit/action.yml b/.github/actions/compile-commit/action.yml index 3cfda30fcc51..d01b0c37fb35 100644 --- a/.github/actions/compile-commit/action.yml +++ b/.github/actions/compile-commit/action.yml @@ -22,7 +22,7 @@ runs: key: compile-commit-success-${{ github.event.pull_request.head.repo.full_name }}-${{ steps.repo-hash.outputs.tree-hash }} - uses: ./.github/actions/setup with: - cleanup-node: false + cleanup-node: true if: steps.check-compile-commit-success.outputs.cache-hit != 'true' - name: Check if a specified commit compiles shell: bash diff --git a/.github/bin/build-matrix-from-impacted.py b/.github/bin/build-matrix-from-impacted.py index 315d73281a4f..56debbae742a 100755 --- a/.github/bin/build-matrix-from-impacted.py +++ b/.github/bin/build-matrix-from-impacted.py @@ -98,6 +98,9 @@ def build(matrix_file, impacted_file, output_file): def check_modules(modules, impacted): if isinstance(modules, str): modules = [modules] + # `impacted` can be empty when GIB detected no changes, but also when GIB was not run at all. + # The latter is the case on builds on master branch. For these builds we want to run all tests, + # so we don't filter out any modules. if impacted and not any(module in impacted for module in modules): return None # concatenate because matrix values should be primitives diff --git a/.github/config/labeler-config.yml b/.github/config/labeler-config.yml index 0d380fe8f99c..4f44778847d5 100644 --- a/.github/config/labeler-config.yml +++ b/.github/config/labeler-config.yml @@ -67,6 +67,10 @@ kafka: - changed-files: - any-glob-to-any-file: 'plugin/trino-kafka/**' +lakehouse: + - changed-files: + - any-glob-to-any-file: 'plugin/trino-lakehouse/**' + loki: - changed-files: - any-glob-to-any-file: 'plugin/trino-loki/**' diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1b458bd70115..abe810f59048 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,13 +4,10 @@ updates: directory: "/" schedule: interval: "weekly" - groups: - dependency-updates: - applies-to: version-updates - update-types: - - major - - minor - - patch - security-updates: - applies-to: security-updates - dependency-type: production + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "daily" + assignees: + - wendigo + open-pull-requests-limit: 10 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c14b305db85..ed492b5913d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,9 +5,6 @@ on: branches: - master pull_request: - paths-ignore: - - 'docs/**' - - '**.md' repository_dispatch: types: [test-with-secrets-command] @@ -50,6 +47,22 @@ concurrency: cancel-in-progress: true jobs: + path-filters: + runs-on: ubuntu-latest + outputs: + docs: ${{ steps.filter.outputs.docs }} + non_docs: ${{ steps.filter.outputs.non_docs }} + steps: + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + id: filter + if: github.event_name == 'pull_request' + with: + # Note: `docs` output is currently unused. The docs changes are tested by maven-checks job, leveraging GIB impact analysis. + # TODO remove use of non_docs filters and remove `path-filters` job. Use GIB impact analysis to guard job skipping. + filters: | + docs: 'docs/**' + non_docs: '!docs/**' + maven-checks: runs-on: ubuntu-latest name: maven-checks ${{ matrix.java-version }} @@ -69,7 +82,7 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: ${{ matrix.cache }} java-version: ${{ matrix.java-version }} @@ -89,6 +102,8 @@ jobs: run: rm -rf ~/.m2/repository/io/trino/trino-* artifact-checks: + needs: path-filters + if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -100,7 +115,7 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: 'restore' cleanup-node: true @@ -122,8 +137,9 @@ jobs: run: core/docker/build.sh check-commits-dispatcher: + needs: path-filters + if: github.event_name == 'pull_request' && needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest - if: github.event_name == 'pull_request' outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: @@ -154,7 +170,7 @@ jobs: check-commit: needs: check-commits-dispatcher runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 if: github.event_name == 'pull_request' && needs.check-commits-dispatcher.outputs.matrix != '' strategy: fail-fast: false @@ -175,6 +191,8 @@ jobs: base_ref: ${{ github.event.pull_request.base.ref }} error-prone-checks: + needs: path-filters + if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -186,7 +204,7 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: restore java-version: 25 @@ -203,6 +221,8 @@ jobs: -pl '!:trino-docs,!:trino-server' test-jdbc-compatibility: + needs: path-filters + if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -214,9 +234,10 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: restore + cleanup-node: 'true' - name: Maven Install run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" @@ -242,6 +263,8 @@ jobs: upload-heap-dump: ${{ env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} hive-tests: + needs: path-filters + if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest strategy: fail-fast: false @@ -259,7 +282,7 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: restore - name: Install Hive Module @@ -298,6 +321,8 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} test-other-modules: + needs: path-filters + if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -309,7 +334,7 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: restore cleanup-node: true @@ -385,6 +410,8 @@ jobs: upload-heap-dump: ${{ env.SECRETS_PRESENT == '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository }} build-test-matrix: + needs: path-filters + if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -397,7 +424,7 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: restore - name: Update PR check @@ -412,7 +439,7 @@ jobs: - name: Maven validate run: | export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -P disable-check-spi-dependencies -pl '!:trino-docs' + $MAVEN validate ${MAVEN_FAST_INSTALL} ${MAVEN_GIB} -Dgib.logImpactedTo=gib-impacted.log -P disable-check-spi-dependencies - name: Set matrix id: set-matrix run: | @@ -489,6 +516,7 @@ jobs: - { modules: plugin/trino-snowflake } - { modules: plugin/trino-snowflake, profile: cloud-tests } - { modules: plugin/trino-sqlserver } + - { modules: plugin/trino-teradata } - { modules: plugin/trino-vertica } - { modules: testing/trino-faulttolerant-tests, profile: default } - { modules: testing/trino-faulttolerant-tests, profile: test-fault-tolerant-delta } @@ -517,10 +545,10 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: restore - cleanup-node: ${{ format('{0}', matrix.modules == 'plugin/trino-singlestore' || matrix.modules == 'plugin/trino-exasol') }} + cleanup-node: ${{ format('{0}', matrix.modules == 'plugin/trino-singlestore' || matrix.modules == 'plugin/trino-exasol' || matrix.modules == 'plugin/trino-oracle') }} java-version: ${{ matrix.jdk != '' && matrix.jdk || '25' }} - name: Maven Install run: | @@ -539,6 +567,7 @@ jobs: && ! (contains(matrix.modules, 'trino-filesystem-gcs') && contains(matrix.profile, 'cloud-tests')) && ! (contains(matrix.modules, 'trino-filesystem-s3') && contains(matrix.profile, 'cloud-tests')) && ! (contains(matrix.modules, 'trino-hdfs') && contains(matrix.profile, 'cloud-tests')) + && ! (contains(matrix.modules, 'trino-teradata')) run: $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} ${{ matrix.profile != '' && format('-P {0}', matrix.profile) || '' }} # Additional tests for selected modules - name: HDFS file system cache isolated JVM tests @@ -764,6 +793,15 @@ jobs: # Cancelled workflows may have left the ephemeral cluster running if: always() run: .github/bin/redshift/delete-aws-redshift.sh + - name: Teradata Tests + id: tests-teradata + env: + CLEARSCAPE_TOKEN: ${{ secrets.CLEARSCAPE_TOKEN }} + CLEARSCAPE_PASSWORD: ${{ secrets.CLEARSCAPE_PASSWORD }} + CLEARSCAPE_REGION: ${{ vars.CLEARSCAPE_REGION }} + if: matrix.modules == 'plugin/trino-teradata' && (env.CLEARSCAPE_TOKEN != '' || env.CLEARSCAPE_PASSWORD != '') + run: | + $MAVEN test ${MAVEN_TEST} -pl :trino-teradata - name: Sanitize artifact name if: always() run: | @@ -808,6 +846,8 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} build-pt: + needs: path-filters + if: github.event_name != 'pull_request' || needs.path-filters.outputs.non_docs == 'true' runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -821,7 +861,7 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: cache: restore cleanup-node: true @@ -852,7 +892,7 @@ jobs: echo "Impacted plugin features:" cat impacted-features.log - name: Product tests artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: product tests and server tarball path: | @@ -877,7 +917,6 @@ jobs: - suite-7-non-generic - suite-hive-transactional - suite-azure - - suite-delta-lake-databricks113 - suite-delta-lake-databricks122 - suite-delta-lake-databricks133 - suite-delta-lake-databricks143 @@ -886,6 +925,7 @@ jobs: - suite-exasol - suite-ranger - suite-gcs + - suite-hive4 - suite-clients - suite-functions - suite-tpch @@ -894,6 +934,7 @@ jobs: - suite-parquet - suite-oauth2 - suite-ldap + - suite-loki - suite-compatibility - suite-all-connectors-smoke - suite-delta-lake-oss @@ -917,9 +958,6 @@ jobs: ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.GCP_CREDENTIALS_KEY != '' }} - - suite: suite-delta-lake-databricks113 - ignore exclusion if: >- - ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} - suite: suite-delta-lake-databricks122 ignore exclusion if: >- ${{ env.CI_SKIP_SECRETS_PRESENCE_CHECKS != '' || secrets.DATABRICKS_TOKEN != '' }} @@ -979,7 +1017,6 @@ jobs: AWS_REGION: "" TRINO_AWS_ACCESS_KEY_ID: "" TRINO_AWS_SECRET_ACCESS_KEY: "" - DATABRICKS_113_JDBC_URL: "" DATABRICKS_122_JDBC_URL: "" DATABRICKS_133_JDBC_URL: "" DATABRICKS_143_JDBC_URL: "" @@ -1024,12 +1061,12 @@ jobs: github.event.client_payload.pull_request.head.sha == github.event.client_payload.slash_command.args.named.sha && format('refs/pull/{0}/head', github.event.client_payload.pull_request.number) || '' }} - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 with: # The job doesn't build anything, so the ~/.m2/repository cache isn't useful cache: 'false' - name: Product tests artifact - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: product tests and server tarball - name: Fix artifact permissions @@ -1052,7 +1089,6 @@ jobs: AWS_REGION: ${{ vars.TRINO_AWS_REGION }} TRINO_AWS_ACCESS_KEY_ID: ${{ vars.TRINO_AWS_ACCESS_KEY_ID }} TRINO_AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }} - DATABRICKS_113_JDBC_URL: ${{ vars.DATABRICKS_113_JDBC_URL }} DATABRICKS_122_JDBC_URL: ${{ vars.DATABRICKS_122_JDBC_URL }} DATABRICKS_133_JDBC_URL: ${{ vars.DATABRICKS_133_JDBC_URL }} DATABRICKS_143_JDBC_URL: ${{ vars.DATABRICKS_143_JDBC_URL }} @@ -1093,3 +1129,38 @@ jobs: check_name: ${{ github.job }} with secrets conclusion: ${{ job.status }} github_token: ${{ secrets.GITHUB_TOKEN }} + + build-success: + if: ${{ always() }} # if `failure()` would not work for cancellations, `!success()` would not work for skipped jobs + runs-on: ubuntu-latest + needs: + - artifact-checks + - build-pt + - build-test-matrix + - check-commit + - check-commits-dispatcher + - error-prone-checks + - hive-tests + - maven-checks + - path-filters + - pt + - test + - test-jdbc-compatibility + - test-other-modules + steps: + - name: "Check results" + run: | + # generated by TestCiWorkflow + echo '${{ needs.artifact-checks.result }}' | grep -xE 'success|skipped' || { echo 'Job "artifact-checks" failed' >&2; exit 1; } + echo '${{ needs.build-pt.result }}' | grep -xE 'success|skipped' || { echo 'Job "build-pt" failed' >&2; exit 1; } + echo '${{ needs.build-test-matrix.result }}' | grep -xE 'success|skipped' || { echo 'Job "build-test-matrix" failed' >&2; exit 1; } + echo '${{ needs.check-commit.result }}' | grep -xE 'success|skipped' || { echo 'Job "check-commit" failed' >&2; exit 1; } + echo '${{ needs.check-commits-dispatcher.result }}' | grep -xE 'success|skipped' || { echo 'Job "check-commits-dispatcher" failed' >&2; exit 1; } + echo '${{ needs.error-prone-checks.result }}' | grep -xE 'success|skipped' || { echo 'Job "error-prone-checks" failed' >&2; exit 1; } + echo '${{ needs.hive-tests.result }}' | grep -xE 'success|skipped' || { echo 'Job "hive-tests" failed' >&2; exit 1; } + echo '${{ needs.maven-checks.result }}' | grep -xE 'success|skipped' || { echo 'Job "maven-checks" failed' >&2; exit 1; } + echo '${{ needs.path-filters.result }}' | grep -xE 'success|skipped' || { echo 'Job "path-filters" failed' >&2; exit 1; } + echo '${{ needs.pt.result }}' | grep -xE 'success|skipped' || { echo 'Job "pt" failed' >&2; exit 1; } + echo '${{ needs.test.result }}' | grep -xE 'success|skipped' || { echo 'Job "test" failed' >&2; exit 1; } + echo '${{ needs.test-jdbc-compatibility.result }}' | grep -xE 'success|skipped' || { echo 'Job "test-jdbc-compatibility" failed' >&2; exit 1; } + echo '${{ needs.test-other-modules.result }}' | grep -xE 'success|skipped' || { echo 'Job "test-other-modules" failed' >&2; exit 1; } diff --git a/.github/workflows/cleanup.yml b/.github/workflows/cleanup.yml index 8a3d99a6d733..21265c5b4f78 100644 --- a/.github/workflows/cleanup.yml +++ b/.github/workflows/cleanup.yml @@ -18,5 +18,5 @@ jobs: # Cancel workflow when PR closed. https://github.com/styfle/cancel-workflow-action#advanced-ignore-sha ignore_sha: true # Note: workflow_id can be a Workflow ID (number) or Workflow File Name (string) or a comma-separated list of those. - workflow_id: "ci.yml,docs.yml" + workflow_id: "ci.yml" access_token: ${{ github.token }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 7f7e9d52b53a..000000000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: docs - -on: - pull_request: - paths: - - 'docs/**' - -defaults: - run: - shell: bash --noprofile --norc -euo pipefail {0} - -env: - # An envar that signals to tests we are executing in the CI environment - CONTINUOUS_INTEGRATION: true - # allow overriding Maven command - MAVEN: ./mvnw - # maven.wagon.rto is in millis, defaults to 30m - MAVEN_OPTS: "-Xmx512M -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000" - MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError -Dmaven.wagon.rto=60000" - MAVEN_FAST_INSTALL: "-B --strict-checksums -V --quiet -T 1C -DskipTests -Dair.check.skip-all" - MAVEN_TEST: "-B --strict-checksums -Dair.check.skip-all --fail-at-end" - RETRY: .github/bin/retry - -# Cancel previous PR builds. -concurrency: - # Cancel all workflow runs except latest within a concurrency group. This is achieved by defining a concurrency group for the PR. - # Non-PR builds have singleton concurrency groups. - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }} - cancel-in-progress: true - -jobs: - path-filters: - runs-on: ubuntu-latest - outputs: - docs: ${{ steps.filter.outputs.docs }} - other: ${{ steps.filter.outputs.other }} - steps: - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 - id: filter - with: - filters: | - docs: 'docs/**' - other: '!docs/**' - - docs-checks: - needs: path-filters - if: ${{ needs.path-filters.outputs.docs == 'true' && needs.path-filters.outputs.other == 'false' }} - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/setup - timeout-minutes: 10 - - name: Maven Checks - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN install -B --strict-checksums -V -T 1C -DskipTests -P ci -am -pl ':trino-docs' - - name: Clean local Maven repo - # Avoid creating a cache entry because this job doesn't download all dependencies - if: steps.cache.outputs.cache-hit != 'true' - run: rm -rf ~/.m2/repository - - test-docs: - needs: path-filters - if: ${{ needs.path-filters.outputs.docs == 'true' && needs.path-filters.outputs.other == 'false' && !contains(github.event.pull_request.labels.*.name, 'release-notes') }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - modules: - - ":trino-main" - - ":trino-plugin-toolkit" - - ":trino-resource-group-managers" - - ":trino-tests" - timeout-minutes: 60 - steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 0 # checkout all commits, as the build result depends on `git describe` equivalent - - uses: ./.github/actions/setup - timeout-minutes: 10 - - name: Maven Install - run: | - export MAVEN_OPTS="${MAVEN_INSTALL_OPTS}" - $RETRY $MAVEN install ${MAVEN_FAST_INSTALL} -am -pl $(echo '${{ matrix.modules }}' | cut -d' ' -f1) - - name: Maven Tests - id: tests - run: $MAVEN test ${MAVEN_TEST} -pl ${{ matrix.modules }} - - name: Sanitize artifact name - if: always() - run: | - # Generate a valid artifact name and make it available to next steps as - # an environment variable ARTIFACT_NAME - # ", :, <, >, |, *, ?, \, / are not allowed in artifact names but we only use : so we remove it - name=$(echo -n "${{ matrix.modules }}" | sed -e 's/[:]//g') - echo "ARTIFACT_NAME=$name" >> $GITHUB_ENV - - name: Upload test results - uses: ./.github/actions/process-test-results - if: always() - with: - artifact-name: ${{ env.ARTIFACT_NAME }} - has-failed-tests: ${{ steps.tests.outcome == 'failure' }} - - name: Clean local Maven repo - # Avoid creating a cache entry because this job doesn't download all dependencies - if: steps.cache.outputs.cache-hit != 'true' - run: rm -rf ~/.m2/repository diff --git a/.github/workflows/milestone.yml b/.github/workflows/milestone.yml index 5ff6c672585d..31fad28c3129 100644 --- a/.github/workflows/milestone.yml +++ b/.github/workflows/milestone.yml @@ -16,11 +16,11 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - uses: ./.github/actions/setup - timeout-minutes: 10 + timeout-minutes: 15 - name: Get milestone from pom.xml run: | .github/bin/retry ./mvnw -v - MILESTONE_NUMBER="$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout | cut -d- -f1)" + MILESTONE_NUMBER="$(./mvnw help:evaluate -Dexpression=project.version -q -DforceStdout --raw-streams | cut -d- -f1)" echo "Setting PR milestone to ${MILESTONE_NUMBER}" echo "MILESTONE_NUMBER=${MILESTONE_NUMBER}" >> $GITHUB_ENV - name: Set milestone to PR diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 04d4ed31adf4..9efd3f3c368e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'trinodb/trino' steps: - - uses: actions/stale@v10.0.0 + - uses: actions/stale@v10.1.0 with: stale-pr-message: 'This pull request has gone a while without any activity. Ask for help on #core-dev on Trino slack.' days-before-pr-stale: 21 diff --git a/README.md b/README.md index 4e8a6b7dda48..9263d3d60bee 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@

Trino download io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/client/trino-client/pom.xml b/client/trino-client/pom.xml index e0bdce83ecf7..e4facbb27cb8 100644 --- a/client/trino-client/pom.xml +++ b/client/trino-client/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/client/trino-client/src/main/java/io/trino/client/DisallowLocalRedirectInterceptor.java b/client/trino-client/src/main/java/io/trino/client/DisallowLocalRedirectInterceptor.java new file mode 100644 index 000000000000..a40dce4c870a --- /dev/null +++ b/client/trino-client/src/main/java/io/trino/client/DisallowLocalRedirectInterceptor.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client; + +import okhttp3.Interceptor; +import okhttp3.Response; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; + +import static java.lang.String.format; + +public class DisallowLocalRedirectInterceptor + implements Interceptor +{ + public DisallowLocalRedirectInterceptor() {} + + @Override + public Response intercept(Chain chain) + throws IOException + { + Response response = chain.proceed(chain.request()); + if (response.isRedirect()) { + String location = response.header("Location"); + if (!redirectAllowed(location)) { + throw new ClientException(format("Following redirect to '%s' is disallowed", location)); + } + } + return response; + } + + boolean redirectAllowed(String location) + { + if (location == null) { + return true; + } + + try { + String host = new URI(location).getHost(); + if (host == null) { + return true; + } + InetAddress[] addresses = InetAddress.getAllByName(host); + for (InetAddress address : addresses) { + if (isLocalAddress(address)) { + return false; + } + } + } + catch (URISyntaxException | UnknownHostException ignored) { + // This will fail later anyway + } + return true; + } + + static boolean isLocalAddress(InetAddress addr) + { + return addr.isAnyLocalAddress() || + addr.isLoopbackAddress() || + addr.isLinkLocalAddress() || + addr.isSiteLocalAddress(); + } +} diff --git a/client/trino-client/src/main/java/io/trino/client/OkHttpSegmentLoader.java b/client/trino-client/src/main/java/io/trino/client/OkHttpSegmentLoader.java index 4a37e298c103..79d9afd5f328 100644 --- a/client/trino-client/src/main/java/io/trino/client/OkHttpSegmentLoader.java +++ b/client/trino-client/src/main/java/io/trino/client/OkHttpSegmentLoader.java @@ -61,10 +61,6 @@ public InputStream load(SpooledSegment segment) .build(); Response response = callFactory.newCall(request).execute(); - if (response.body() == null) { - throw new IOException("Could not open segment for streaming, got empty body"); - } - if (response.isSuccessful()) { return response.body().byteStream(); } diff --git a/client/trino-client/src/main/java/io/trino/client/ProtocolHeaders.java b/client/trino-client/src/main/java/io/trino/client/ProtocolHeaders.java index 69e19da4184f..015ea8953981 100644 --- a/client/trino-client/src/main/java/io/trino/client/ProtocolHeaders.java +++ b/client/trino-client/src/main/java/io/trino/client/ProtocolHeaders.java @@ -17,6 +17,40 @@ import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_CATALOG; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_CLIENT_CAPABILITIES; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_CLIENT_INFO; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_CLIENT_TAGS; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_EXTRA_CREDENTIAL; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_LANGUAGE; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_ORIGINAL_ROLES; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_ORIGINAL_USER; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_PATH; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_PREPARED_STATEMENT; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_QUERY_DATA_ENCODING; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_RESOURCE_ESTIMATE; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_ROLE; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_SCHEMA; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_SESSION; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_SOURCE; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_TIME_ZONE; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_TRACE_TOKEN; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_TRANSACTION_ID; +import static io.trino.client.ProtocolHeaders.Headers.REQUEST_USER; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_ADDED_PREPARE; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_CLEAR_SESSION; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_CLEAR_TRANSACTION_ID; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_DEALLOCATED_PREPARE; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_QUERY_DATA_ENCODING; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_RESET_AUTHORIZATION_USER; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_SET_AUTHORIZATION_USER; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_SET_CATALOG; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_SET_ORIGINAL_ROLES; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_SET_PATH; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_SET_ROLE; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_SET_SCHEMA; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_SET_SESSION; +import static io.trino.client.ProtocolHeaders.Headers.RESPONSE_STARTED_TRANSACTION_ID; import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; @@ -24,6 +58,56 @@ public final class ProtocolHeaders { public static final ProtocolHeaders TRINO_HEADERS = new ProtocolHeaders("Trino"); + enum Headers + { + REQUEST_USER("User"), + REQUEST_ORIGINAL_USER("Original-User"), + REQUEST_ORIGINAL_ROLES("Original-Roles"), + REQUEST_SOURCE("Source"), + REQUEST_CATALOG("Catalog"), + REQUEST_SCHEMA("Schema"), + REQUEST_PATH("Path"), + REQUEST_TIME_ZONE("Time-Zone"), + REQUEST_LANGUAGE("Language"), + REQUEST_TRACE_TOKEN("Trace-Token"), + REQUEST_SESSION("Session"), + REQUEST_ROLE("Role"), + REQUEST_PREPARED_STATEMENT("Prepared-Statement"), + REQUEST_TRANSACTION_ID("Transaction-Id"), + REQUEST_CLIENT_INFO("Client-Info"), + REQUEST_CLIENT_TAGS("Client-Tags"), + REQUEST_CLIENT_CAPABILITIES("Client-Capabilities"), + REQUEST_RESOURCE_ESTIMATE("Resource-Estimate"), + REQUEST_EXTRA_CREDENTIAL("Extra-Credential"), + REQUEST_QUERY_DATA_ENCODING("Query-Data-Encoding"), + RESPONSE_SET_CATALOG("Set-Catalog"), + RESPONSE_SET_SCHEMA("Set-Schema"), + RESPONSE_SET_PATH("Set-Path"), + RESPONSE_SET_SESSION("Set-Session"), + RESPONSE_CLEAR_SESSION("Clear-Session"), + RESPONSE_SET_ROLE("Set-Role"), + RESPONSE_SET_ORIGINAL_ROLES("Set-Original-Roles"), + RESPONSE_QUERY_DATA_ENCODING("Query-Data-Encoding"), + RESPONSE_ADDED_PREPARE("Added-Prepare"), + RESPONSE_DEALLOCATED_PREPARE("Deallocated-Prepare"), + RESPONSE_STARTED_TRANSACTION_ID("Started-Transaction-Id"), + RESPONSE_CLEAR_TRANSACTION_ID("Clear-Transaction-Id"), + RESPONSE_SET_AUTHORIZATION_USER("Set-Authorization-User"), + RESPONSE_RESET_AUTHORIZATION_USER("Reset-Authorization-User"); + + private final String headerName; + + Headers(final String headerName) + { + this.headerName = requireNonNull(headerName, "headerName is null"); + } + + public String withProtocolName(String protocolName) + { + return "X-" + protocolName + "-" + headerName; + } + } + private final String name; private final String requestUser; private final String requestOriginalUser; @@ -74,41 +158,40 @@ private ProtocolHeaders(String name) requireNonNull(name, "name is null"); checkArgument(!name.isEmpty(), "name is empty"); this.name = name; - String prefix = "X-" + name + "-"; - requestUser = prefix + "User"; - requestOriginalUser = prefix + "Original-User"; - requestOriginalRole = prefix + "Original-Roles"; - requestSource = prefix + "Source"; - requestCatalog = prefix + "Catalog"; - requestSchema = prefix + "Schema"; - requestPath = prefix + "Path"; - requestTimeZone = prefix + "Time-Zone"; - requestLanguage = prefix + "Language"; - requestTraceToken = prefix + "Trace-Token"; - requestSession = prefix + "Session"; - requestRole = prefix + "Role"; - requestPreparedStatement = prefix + "Prepared-Statement"; - requestTransactionId = prefix + "Transaction-Id"; - requestClientInfo = prefix + "Client-Info"; - requestClientTags = prefix + "Client-Tags"; - requestClientCapabilities = prefix + "Client-Capabilities"; - requestResourceEstimate = prefix + "Resource-Estimate"; - requestExtraCredential = prefix + "Extra-Credential"; - requestQueryDataEncoding = prefix + "Query-Data-Encoding"; - responseSetCatalog = prefix + "Set-Catalog"; - responseSetSchema = prefix + "Set-Schema"; - responseSetPath = prefix + "Set-Path"; - responseSetSession = prefix + "Set-Session"; - responseClearSession = prefix + "Clear-Session"; - responseSetRole = prefix + "Set-Role"; - responseQueryDataEncoding = prefix + "Query-Data-Encoding"; - responseAddedPrepare = prefix + "Added-Prepare"; - responseDeallocatedPrepare = prefix + "Deallocated-Prepare"; - responseStartedTransactionId = prefix + "Started-Transaction-Id"; - responseClearTransactionId = prefix + "Clear-Transaction-Id"; - responseSetAuthorizationUser = prefix + "Set-Authorization-User"; - responseResetAuthorizationUser = prefix + "Reset-Authorization-User"; - responseOriginalRole = prefix + "Set-Original-Roles"; + requestUser = REQUEST_USER.withProtocolName(name); + requestOriginalUser = REQUEST_ORIGINAL_USER.withProtocolName(name); + requestOriginalRole = REQUEST_ORIGINAL_ROLES.withProtocolName(name); + requestSource = REQUEST_SOURCE.withProtocolName(name); + requestCatalog = REQUEST_CATALOG.withProtocolName(name); + requestSchema = REQUEST_SCHEMA.withProtocolName(name); + requestPath = REQUEST_PATH.withProtocolName(name); + requestTimeZone = REQUEST_TIME_ZONE.withProtocolName(name); + requestLanguage = REQUEST_LANGUAGE.withProtocolName(name); + requestTraceToken = REQUEST_TRACE_TOKEN.withProtocolName(name); + requestSession = REQUEST_SESSION.withProtocolName(name); + requestRole = REQUEST_ROLE.withProtocolName(name); + requestPreparedStatement = REQUEST_PREPARED_STATEMENT.withProtocolName(name); + requestTransactionId = REQUEST_TRANSACTION_ID.withProtocolName(name); + requestClientInfo = REQUEST_CLIENT_INFO.withProtocolName(name); + requestClientTags = REQUEST_CLIENT_TAGS.withProtocolName(name); + requestClientCapabilities = REQUEST_CLIENT_CAPABILITIES.withProtocolName(name); + requestResourceEstimate = REQUEST_RESOURCE_ESTIMATE.withProtocolName(name); + requestExtraCredential = REQUEST_EXTRA_CREDENTIAL.withProtocolName(name); + requestQueryDataEncoding = REQUEST_QUERY_DATA_ENCODING.withProtocolName(name); + responseSetCatalog = RESPONSE_SET_CATALOG.withProtocolName(name); + responseSetSchema = RESPONSE_SET_SCHEMA.withProtocolName(name); + responseSetPath = RESPONSE_SET_PATH.withProtocolName(name); + responseSetSession = RESPONSE_SET_SESSION.withProtocolName(name); + responseClearSession = RESPONSE_CLEAR_SESSION.withProtocolName(name); + responseSetRole = RESPONSE_SET_ROLE.withProtocolName(name); + responseQueryDataEncoding = RESPONSE_QUERY_DATA_ENCODING.withProtocolName(name); + responseAddedPrepare = RESPONSE_ADDED_PREPARE.withProtocolName(name); + responseDeallocatedPrepare = RESPONSE_DEALLOCATED_PREPARE.withProtocolName(name); + responseStartedTransactionId = RESPONSE_STARTED_TRANSACTION_ID.withProtocolName(name); + responseClearTransactionId = RESPONSE_CLEAR_TRANSACTION_ID.withProtocolName(name); + responseSetAuthorizationUser = RESPONSE_SET_AUTHORIZATION_USER.withProtocolName(name); + responseResetAuthorizationUser = RESPONSE_RESET_AUTHORIZATION_USER.withProtocolName(name); + responseOriginalRole = RESPONSE_SET_ORIGINAL_ROLES.withProtocolName(name); } public String getProtocolName() diff --git a/client/trino-client/src/main/java/io/trino/client/uri/ConnectionProperties.java b/client/trino-client/src/main/java/io/trino/client/uri/ConnectionProperties.java index c8c1b67b5f95..4da7f640c16d 100644 --- a/client/trino-client/src/main/java/io/trino/client/uri/ConnectionProperties.java +++ b/client/trino-client/src/main/java/io/trino/client/uri/ConnectionProperties.java @@ -116,6 +116,7 @@ enum SslVerificationMode public static final ConnectionProperty> RESOURCE_ESTIMATES = new ResourceEstimates(); public static final ConnectionProperty> SQL_PATH = new SqlPath(); public static final ConnectionProperty VALIDATE_CONNECTION = new ValidateConnection(); + public static final ConnectionProperty DISALLOW_LOCAL_REDIRECT = new LocalRedirectDisallowed(); private static final Set> ALL_PROPERTIES = ImmutableSet.>builder() // Keep sorted @@ -128,6 +129,7 @@ enum SslVerificationMode .add(CLIENT_INFO) .add(CLIENT_TAGS) .add(DISABLE_COMPRESSION) + .add(DISALLOW_LOCAL_REDIRECT) .add(DNS_RESOLVER) .add(DNS_RESOLVER_CONTEXT) .add(ENCODING) @@ -958,6 +960,15 @@ public AssumeNullCatalogMeansCurrentCatalog() } } + private static class LocalRedirectDisallowed + extends AbstractConnectionProperty + { + public LocalRedirectDisallowed() + { + super(PropertyName.DISALLOW_LOCAL_REDIRECT, Optional.of(false), NOT_REQUIRED, ALLOWED, BOOLEAN_CONVERTER); + } + } + private static class MapPropertyParser { private static final CharMatcher PRINTABLE_ASCII = CharMatcher.inRange((char) 0x21, (char) 0x7E); diff --git a/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java b/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java index 796a42b40605..b882792c146d 100644 --- a/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java +++ b/client/trino-client/src/main/java/io/trino/client/uri/HttpClientFactory.java @@ -14,6 +14,7 @@ package io.trino.client.uri; import io.trino.client.ClientException; +import io.trino.client.DisallowLocalRedirectInterceptor; import io.trino.client.DnsResolver; import io.trino.client.auth.external.CompositeRedirectHandler; import io.trino.client.auth.external.ExternalAuthenticator; @@ -125,6 +126,10 @@ public static OkHttpClient.Builder unauthenticatedClientBuilder(TrinoUri uri, St setupTimeouts(builder, toIntExact(uri.getTimeout().toMillis()), TimeUnit.MILLISECONDS); setupHttpLogging(builder, uri.getHttpLoggingLevel()); + if (uri.isLocalRedirectDisallowed()) { + builder.addNetworkInterceptor(new DisallowLocalRedirectInterceptor()); + } + if (uri.isUseSecureConnection()) { ConnectionProperties.SslVerificationMode sslVerificationMode = uri.getSslVerification(); if (sslVerificationMode.equals(FULL) || sslVerificationMode.equals(CA)) { diff --git a/client/trino-client/src/main/java/io/trino/client/uri/PropertyName.java b/client/trino-client/src/main/java/io/trino/client/uri/PropertyName.java index b0f13b3ac096..1a7de8011bb3 100644 --- a/client/trino-client/src/main/java/io/trino/client/uri/PropertyName.java +++ b/client/trino-client/src/main/java/io/trino/client/uri/PropertyName.java @@ -76,6 +76,7 @@ public enum PropertyName TIMEZONE("timezone"), TRACE_TOKEN("traceToken"), USER("user"), + DISALLOW_LOCAL_REDIRECT("disallowLocalRedirect"), VALIDATE_CONNECTION("validateConnection"); private final String key; diff --git a/client/trino-client/src/main/java/io/trino/client/uri/TrinoUri.java b/client/trino-client/src/main/java/io/trino/client/uri/TrinoUri.java index 8b10dbe76586..15e4e9b3f0b5 100644 --- a/client/trino-client/src/main/java/io/trino/client/uri/TrinoUri.java +++ b/client/trino-client/src/main/java/io/trino/client/uri/TrinoUri.java @@ -51,6 +51,7 @@ import static io.trino.client.uri.ConnectionProperties.CLIENT_INFO; import static io.trino.client.uri.ConnectionProperties.CLIENT_TAGS; import static io.trino.client.uri.ConnectionProperties.DISABLE_COMPRESSION; +import static io.trino.client.uri.ConnectionProperties.DISALLOW_LOCAL_REDIRECT; import static io.trino.client.uri.ConnectionProperties.DNS_RESOLVER; import static io.trino.client.uri.ConnectionProperties.DNS_RESOLVER_CONTEXT; import static io.trino.client.uri.ConnectionProperties.ENCODING; @@ -423,6 +424,11 @@ public boolean isCompressionDisabled() return resolveWithDefault(DISABLE_COMPRESSION, false); } + public boolean isLocalRedirectDisallowed() + { + return resolveWithDefault(DISALLOW_LOCAL_REDIRECT, false); + } + public Optional getEncoding() { Optional encodings = resolveOptional(ENCODING); @@ -1058,6 +1064,11 @@ public Builder setValidateConnection(boolean value) return setProperty(VALIDATE_CONNECTION, value); } + public Builder setDisallowLocalRedirect(boolean value) + { + return setProperty(DISALLOW_LOCAL_REDIRECT, value); + } + Builder setProperty(ConnectionProperty connectionProperty, T value) { properties.put(connectionProperty.getKey(), connectionProperty.encodeValue(value)); diff --git a/client/trino-client/src/test/java/io/trino/client/TestDisallowLocalRedirectInterceptor.java b/client/trino-client/src/test/java/io/trino/client/TestDisallowLocalRedirectInterceptor.java new file mode 100644 index 000000000000..ea27d38d983e --- /dev/null +++ b/client/trino-client/src/test/java/io/trino/client/TestDisallowLocalRedirectInterceptor.java @@ -0,0 +1,159 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.client; + +import okhttp3.Interceptor; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestDisallowLocalRedirectInterceptor +{ + private static final String BASE_URL = "https://example.com"; + + @Test + public void testRedirectValidation() + throws IOException + { + DisallowLocalRedirectInterceptor redirector = new DisallowLocalRedirectInterceptor(); + + // Valid external URIs + assertThat(redirector.intercept(chainWithRedirectLocation("https://www.example.com"))) + .isNotNull(); + + assertThat(redirector.intercept(chainWithRedirectLocation("https://api.github.com"))) + .isNotNull(); + + // Invalid URI + assertThat(redirector.intercept(chainWithRedirectLocation("not a valid uri"))) + .isNotNull(); + + // Unresolvable host + assertThat(redirector.intercept(chainWithRedirectLocation("https://nonexistent.example.invalid"))) + .isNotNull(); + + // Local URIs + assertThatThrownBy(() -> redirector.intercept(chainWithRedirectLocation("https://127.0.0.1"))) + .isInstanceOf(ClientException.class) + .hasMessage("Following redirect to 'https://127.0.0.1' is disallowed"); + + assertThatThrownBy(() -> redirector.intercept(chainWithRedirectLocation("https://localhost"))) + .isInstanceOf(ClientException.class) + .hasMessage("Following redirect to 'https://localhost' is disallowed"); + + assertThatThrownBy(() -> redirector.intercept(chainWithRedirectLocation("http://192.168.1.1"))) + .isInstanceOf(ClientException.class) + .hasMessage("Following redirect to 'http://192.168.1.1' is disallowed"); + + assertThatThrownBy(() -> redirector.intercept(chainWithRedirectLocation("https://0.0.0.0"))) + .isInstanceOf(ClientException.class) + .hasMessage("Following redirect to 'https://0.0.0.0' is disallowed"); + + assertThatThrownBy(() -> redirector.intercept(chainWithRedirectLocation("https://172.16.0.1/uri"))) + .isInstanceOf(ClientException.class) + .hasMessage("Following redirect to 'https://172.16.0.1/uri' is disallowed"); + + assertThatThrownBy(() -> redirector.intercept(chainWithRedirectLocation("https://169.254.169.254"))) + .isInstanceOf(ClientException.class) + .hasMessage("Following redirect to 'https://169.254.169.254' is disallowed"); + } + + private static Interceptor.Chain chainWithRedirectLocation(String location) + { + return new TestingInterceptorChain(new Response.Builder() + .request(new Request.Builder().url(BASE_URL).build()) + .protocol(Protocol.HTTP_1_1) + .code(302) + .message("Found") + .header("Location", location) + .build()); + } + + private static class TestingInterceptorChain + implements Interceptor.Chain + { + private final Response response; + + TestingInterceptorChain(Response response) + { + this.response = response; + } + + @Override + public Request request() + { + return response.request(); + } + + @Override + public Response proceed(Request request) + { + return response; + } + + @Override + public int connectTimeoutMillis() + { + throw new UnsupportedOperationException(); + } + + @Override + public Interceptor.Chain withConnectTimeout(int timeout, java.util.concurrent.TimeUnit unit) + { + throw new UnsupportedOperationException(); + } + + @Override + public int readTimeoutMillis() + { + throw new UnsupportedOperationException(); + } + + @Override + public Interceptor.Chain withReadTimeout(int timeout, java.util.concurrent.TimeUnit unit) + { + throw new UnsupportedOperationException(); + } + + @Override + public int writeTimeoutMillis() + { + throw new UnsupportedOperationException(); + } + + @Override + public Interceptor.Chain withWriteTimeout(int timeout, java.util.concurrent.TimeUnit unit) + { + throw new UnsupportedOperationException(); + } + + @Override + public okhttp3.Connection connection() + { + throw new UnsupportedOperationException(); + } + + @Override + public okhttp3.Call call() + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/client/trino-jdbc/pom.xml b/client/trino-jdbc/pom.xml index dec7bd25e66a..c2ee67c1e9cf 100644 --- a/client/trino-jdbc/pom.xml +++ b/client/trino-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -345,19 +345,19 @@ org.testcontainers - oracle-xe + testcontainers test org.testcontainers - postgresql + testcontainers-oracle-free test org.testcontainers - testcontainers + testcontainers-postgresql test @@ -420,6 +420,21 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/client/trino-jdbc/src/main/java/io/trino/jdbc/AsyncResultIterator.java b/client/trino-jdbc/src/main/java/io/trino/jdbc/AsyncResultIterator.java index 24fa9d1ebc9c..cf72441ebb25 100644 --- a/client/trino-jdbc/src/main/java/io/trino/jdbc/AsyncResultIterator.java +++ b/client/trino-jdbc/src/main/java/io/trino/jdbc/AsyncResultIterator.java @@ -75,7 +75,7 @@ public class AsyncResultIterator warningsManager.addWarnings(results.getWarnings()); for (List row : client.currentRows()) { rowQueue.put(row); - if (rowsProcessed++ % BATCH_SIZE == 0) { + if (++rowsProcessed == BATCH_SIZE) { semaphore.release(rowsProcessed); rowsProcessed = 0; } diff --git a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestJdbcVendorCompatibility.java b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestJdbcVendorCompatibility.java index 395cc792d00d..5899987a47fd 100644 --- a/client/trino-jdbc/src/test/java/io/trino/jdbc/TestJdbcVendorCompatibility.java +++ b/client/trino-jdbc/src/test/java/io/trino/jdbc/TestJdbcVendorCompatibility.java @@ -24,8 +24,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.parallel.Execution; -import org.testcontainers.containers.OracleContainer; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.oracle.OracleContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import java.io.Closeable; import java.sql.Connection; @@ -539,7 +539,7 @@ private static class OracleReferenceDriver OracleReferenceDriver() { - oracleServer = new OracleContainer("gvenzl/oracle-xe:11.2.0.2-full") + oracleServer = new OracleContainer("gvenzl/oracle-free:23.9-slim") .usingSid(); oracleServer.start(); } @@ -618,7 +618,7 @@ public String toString() private static class PostgresqlReferenceDriver implements ReferenceDriver { - private final PostgreSQLContainer postgresqlContainer; + private final PostgreSQLContainer postgresqlContainer; private Connection connection; private Statement statement; private Optional> timezoneSet = Optional.empty(); @@ -626,7 +626,7 @@ private static class PostgresqlReferenceDriver PostgresqlReferenceDriver() { // Use the current latest PostgreSQL version as the reference - postgresqlContainer = new PostgreSQLContainer<>("postgres:15"); + postgresqlContainer = new PostgreSQLContainer("postgres:15"); postgresqlContainer.start(); } diff --git a/core/.temurin-release b/core/.temurin-release new file mode 100644 index 000000000000..2a0efb7169a7 --- /dev/null +++ b/core/.temurin-release @@ -0,0 +1 @@ +jdk-25.0.1+8 diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index f9c1343b1454..777e8c012357 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -34,8 +34,8 @@ RUN \ mkdir -p /tmp/overlay/usr/libexec/ && \ touch /tmp/overlay/usr/libexec/grepconf.sh && \ chmod +x /tmp/overlay/usr/libexec/grepconf.sh && \ - yum update -y && \ - yum install --installroot /tmp/overlay --setopt install_weak_deps=false --nodocs -y \ + dnf update -y && \ + dnf install --installroot /tmp/overlay --setopt install_weak_deps=false --nodocs -y \ less \ libstdc++ `# required by snappy and duckdb` \ curl-minimal grep `# required by health-check` \ diff --git a/core/docker/build.sh b/core/docker/build.sh index b0f1f0b6a875..c0a5b51e2ac2 100755 --- a/core/docker/build.sh +++ b/core/docker/build.sh @@ -28,8 +28,8 @@ TRINO_VERSION= TAG_PREFIX=trino SERVER_ARTIFACT=trino-server -JDK_RELEASE=$(cat "${SOURCE_DIR}/core/jdk/current") -JDKS_PATH="${SOURCE_DIR}/core/jdk" +TEMURIN_RELEASE=$(cat "${SOURCE_DIR}/core/.temurin-release") +TEMURIN_DOWNLOAD_URL="https://api.adoptium.net/v3/binary/version/{release_name}/linux/{arch}/jdk/hotspot/normal/eclipse?project=jdk" SKIP_TESTS=false @@ -72,10 +72,6 @@ while getopts ":a:h:r:p:t:j:x" o; do done shift $((OPTIND - 1)) -function prop { - grep "^${1}=" "${2}" | cut -d'=' -f2- -} - function check_environment() { if ! command -v jq &> /dev/null; then echo >&2 "Please install jq" @@ -83,16 +79,22 @@ function check_environment() { fi } -function jdk_download_link() { - local RELEASE_PATH="${1}" +function temurin_download_uri() { + local RELEASE_NAME="${1}" local ARCH="${2}" - if [ -f "${RELEASE_PATH}/${ARCH}" ]; then - prop "distributionUrl" "${RELEASE_PATH}/${ARCH}" - else - echo "${ARCH} is not supported for JDK release ${RELEASE_PATH}" - exit 1 - fi + case "${ARCH}" in + "arm64") + echo "${TEMURIN_DOWNLOAD_URL}" | sed -e "s/{release_name}/${RELEASE_NAME}/" -e "s/{arch}/aarch64/" ;; + "amd64") + echo "${TEMURIN_DOWNLOAD_URL}" | sed -e "s/{release_name}/${RELEASE_NAME}/" -e "s/{arch}/x64/" ;; + "ppc64le") + echo "${TEMURIN_DOWNLOAD_URL}" | sed -e "s/{release_name}/${RELEASE_NAME}/" -e "s/{arch}/ppc64le/" ;; + *) + echo "Unsupported architecture: ${ARCH}" >&2 + exit 1 + ;; + esac } check_environment @@ -102,12 +104,12 @@ if [ -n "$TRINO_VERSION" ]; then for artifactId in "io.trino:${SERVER_ARTIFACT}:${TRINO_VERSION}:tar.gz" io.trino:trino-cli:"${TRINO_VERSION}":jar:executable; do "${SOURCE_DIR}/mvnw" -C dependency:get -Dtransitive=false -Dartifact="$artifactId" done - local_repo=$("${SOURCE_DIR}/mvnw" -B help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) + local_repo=$("${SOURCE_DIR}/mvnw" -B help:evaluate -Dexpression=settings.localRepository -q -DforceStdout --raw-streams) trino_server="$local_repo/io/trino/${SERVER_ARTIFACT}/${TRINO_VERSION}/${SERVER_ARTIFACT}-${TRINO_VERSION}.tar.gz" trino_client="$local_repo/io/trino/trino-cli/${TRINO_VERSION}/trino-cli-${TRINO_VERSION}-executable.jar" chmod +x "$trino_client" else - TRINO_VERSION=$("${SOURCE_DIR}/mvnw" -f "${SOURCE_DIR}/pom.xml" --quiet help:evaluate -Dexpression=project.version -DforceStdout) + TRINO_VERSION=$("${SOURCE_DIR}/mvnw" -f "${SOURCE_DIR}/pom.xml" --quiet help:evaluate -Dexpression=project.version -DforceStdout --raw-streams) echo "🎯 Using currently built artifacts from the core/${SERVER_ARTIFACT} and client/trino-cli modules and version ${TRINO_VERSION}" trino_server="${SOURCE_DIR}/core/${SERVER_ARTIFACT}/target/${SERVER_ARTIFACT}-${TRINO_VERSION}.tar.gz" trino_client="${SOURCE_DIR}/client/trino-cli/target/trino-cli-${TRINO_VERSION}-executable.jar" @@ -129,14 +131,15 @@ fi TAG="${TAG_PREFIX}:${TRINO_VERSION}" for arch in "${ARCHITECTURES[@]}"; do - echo "🫙 Building the image for $arch with JDK ${JDK_RELEASE}" + JDK_DOWNLOAD_LINK="$(temurin_download_uri "${TEMURIN_RELEASE}" "${arch}")" + echo "🫙 Building the image for $arch with JDK ${JDK_DOWNLOAD_LINK}" docker build \ "${WORK_DIR}" \ --progress=plain \ --pull \ --build-arg ARCH="${arch}" \ - --build-arg JDK_VERSION="${JDK_RELEASE}" \ - --build-arg JDK_DOWNLOAD_LINK="$(jdk_download_link "${JDKS_PATH}/${JDK_RELEASE}" "${arch}")" \ + --build-arg JDK_VERSION="${TEMURIN_RELEASE}" \ + --build-arg JDK_DOWNLOAD_LINK="${JDK_DOWNLOAD_LINK}" \ --platform "linux/$arch" \ -f Dockerfile \ -t "${TAG}-$arch" diff --git a/core/jdk/current b/core/jdk/current deleted file mode 100644 index 3237184257ef..000000000000 --- a/core/jdk/current +++ /dev/null @@ -1 +0,0 @@ -temurin/jdk-25+36 diff --git a/core/jdk/temurin/jdk-25+36/amd64 b/core/jdk/temurin/jdk-25+36/amd64 deleted file mode 100644 index 12bc6dc19109..000000000000 --- a/core/jdk/temurin/jdk-25+36/amd64 +++ /dev/null @@ -1 +0,0 @@ -distributionUrl=https://api.adoptium.net/v3/binary/version/jdk-25+36/linux/x64/jdk/hotspot/normal/eclipse?project=jdk diff --git a/core/jdk/temurin/jdk-25+36/arm64 b/core/jdk/temurin/jdk-25+36/arm64 deleted file mode 100644 index 5afc268393c4..000000000000 --- a/core/jdk/temurin/jdk-25+36/arm64 +++ /dev/null @@ -1 +0,0 @@ -distributionUrl=https://api.adoptium.net/v3/binary/version/jdk-25+36/linux/aarch64/jdk/hotspot/normal/eclipse?project=jdk diff --git a/core/jdk/temurin/jdk-25+36/ppc64le b/core/jdk/temurin/jdk-25+36/ppc64le deleted file mode 100644 index efa5eba13851..000000000000 --- a/core/jdk/temurin/jdk-25+36/ppc64le +++ /dev/null @@ -1 +0,0 @@ -distributionUrl=https://api.adoptium.net/v3/binary/version/jdk-25+36/linux/ppc64le/jdk/hotspot/normal/eclipse?project=jdk diff --git a/core/trino-grammar/pom.xml b/core/trino-grammar/pom.xml index d275b4be3712..1bc8902167db 100644 --- a/core/trino-grammar/pom.xml +++ b/core/trino-grammar/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index 507382809dca..0825d970b6de 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -73,7 +73,6 @@ com.nimbusds oauth2-oidc-sdk - jdk11 @@ -351,7 +350,7 @@ org.apache.lucene lucene-analysis-common - 10.3.0 + 10.3.1 @@ -547,13 +546,13 @@ org.testcontainers - postgresql + testcontainers test org.testcontainers - testcontainers + testcontainers-postgresql test diff --git a/core/trino-main/src/main/java/io/trino/Session.java b/core/trino-main/src/main/java/io/trino/Session.java index 7bf76ce504b8..033d68f01757 100644 --- a/core/trino-main/src/main/java/io/trino/Session.java +++ b/core/trino-main/src/main/java/io/trino/Session.java @@ -541,7 +541,7 @@ public ConnectorSession toConnectorSession(CatalogHandle catalogHandle) public SessionRepresentation toSessionRepresentation() { return new SessionRepresentation( - queryId.toString(), + queryId.id(), querySpan, transactionId, clientTransactionSupport, diff --git a/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java b/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java index de76446c7ce7..e2fdd1767b6b 100644 --- a/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java +++ b/core/trino-main/src/main/java/io/trino/SystemSessionProperties.java @@ -798,6 +798,11 @@ public SystemSessionProperties( "Retry policy", RetryPolicy.class, queryManagerConfig.getRetryPolicy(), + value -> { + if (!queryManagerConfig.getAllowedRetryPolicies().contains(value)) { + throw new TrinoException(INVALID_SESSION_PROPERTY, format("Retry policy %s not allowed. Must be one of %s", value, queryManagerConfig.getAllowedRetryPolicies())); + } + }, true), integerProperty( QUERY_RETRY_ATTEMPTS, diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogConnector.java b/core/trino-main/src/main/java/io/trino/connector/CatalogConnector.java index 92140bb63879..0fbee8e14d00 100644 --- a/core/trino-main/src/main/java/io/trino/connector/CatalogConnector.java +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogConnector.java @@ -21,6 +21,7 @@ import java.util.Optional; import static com.google.common.base.MoreObjects.toStringHelper; +import static io.trino.metadata.CatalogStatus.OPERATIONAL; import static java.util.Objects.requireNonNull; public class CatalogConnector @@ -55,7 +56,7 @@ public CatalogConnector( catalogConnector, informationSchemaConnector, systemConnector, - true); + OPERATIONAL); } public CatalogHandle getCatalogHandle() diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogPruneTask.java b/core/trino-main/src/main/java/io/trino/connector/CatalogPruneTask.java index fbad249b5db7..3dc6b650d0bc 100644 --- a/core/trino-main/src/main/java/io/trino/connector/CatalogPruneTask.java +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogPruneTask.java @@ -137,8 +137,8 @@ public void pruneWorkerCatalogs() List activeCatalogs = getActiveCatalogs(); pruneWorkerCatalogs(online, activeCatalogs); - // prune all inactive catalogs - we pass an empty set here because manager always retains active catalogs - connectorServicesProvider.pruneCatalogs(ImmutableSet.of()); + // prune inactive catalogs locally + connectorServicesProvider.pruneCatalogs(ImmutableSet.copyOf(activeCatalogs)); } void pruneWorkerCatalogs(Set online, List activeCatalogs) diff --git a/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java b/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java index 7255badaeecf..d4dd889f91ec 100644 --- a/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java +++ b/core/trino-main/src/main/java/io/trino/connector/ConnectorServices.java @@ -181,7 +181,7 @@ public ConnectorServices(Tracer tracer, CatalogHandle catalogHandle, Connector c this.accessControl = Optional.ofNullable(accessControl); List> sessionProperties = connector.getSessionProperties(); - requireNonNull(sessionProperties, format("Connector '%s' returned a null system properties set", catalogHandle)); + requireNonNull(sessionProperties, format("Connector '%s' returned a null session properties set", catalogHandle)); this.sessionProperties = Maps.uniqueIndex(sessionProperties, PropertyMetadata::getName); List> tableProperties = connector.getTableProperties(); diff --git a/core/trino-main/src/main/java/io/trino/connector/DynamicCatalogManagerModule.java b/core/trino-main/src/main/java/io/trino/connector/DynamicCatalogManagerModule.java index 4dbe6a975dbc..3eb104ac26bf 100644 --- a/core/trino-main/src/main/java/io/trino/connector/DynamicCatalogManagerModule.java +++ b/core/trino-main/src/main/java/io/trino/connector/DynamicCatalogManagerModule.java @@ -18,6 +18,7 @@ import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.units.Duration; +import io.trino.connector.WorkerDynamicCatalogManager.NoOpWorkerCatalogManager; import io.trino.connector.system.GlobalSystemConnector; import io.trino.metadata.CatalogManager; import io.trino.server.ServerConfig; @@ -51,6 +52,7 @@ protected void setup(Binder binder) }).build()); } else { + binder.bind(CatalogManager.class).to(NoOpWorkerCatalogManager.class).in(Scopes.SINGLETON); binder.bind(WorkerDynamicCatalogManager.class).in(Scopes.SINGLETON); binder.bind(ConnectorServicesProvider.class).to(WorkerDynamicCatalogManager.class).in(Scopes.SINGLETON); // catalog manager is not registered on worker diff --git a/core/trino-main/src/main/java/io/trino/connector/FileCatalogStoreFactory.java b/core/trino-main/src/main/java/io/trino/connector/FileCatalogStoreFactory.java index 819d6f656eaf..131c46937703 100644 --- a/core/trino-main/src/main/java/io/trino/connector/FileCatalogStoreFactory.java +++ b/core/trino-main/src/main/java/io/trino/connector/FileCatalogStoreFactory.java @@ -36,6 +36,7 @@ public CatalogStore create(Map config) Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/core/trino-main/src/main/java/io/trino/connector/WorkerDynamicCatalogManager.java b/core/trino-main/src/main/java/io/trino/connector/WorkerDynamicCatalogManager.java index b3bef6b41806..dcd3d763d763 100644 --- a/core/trino-main/src/main/java/io/trino/connector/WorkerDynamicCatalogManager.java +++ b/core/trino-main/src/main/java/io/trino/connector/WorkerDynamicCatalogManager.java @@ -21,8 +21,11 @@ import io.airlift.log.Logger; import io.trino.Session; import io.trino.connector.system.GlobalSystemConnector; +import io.trino.metadata.Catalog; +import io.trino.metadata.CatalogManager; import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.spi.TrinoException; +import io.trino.spi.catalog.CatalogName; import io.trino.spi.catalog.CatalogProperties; import io.trino.spi.connector.ConnectorName; import jakarta.annotation.PreDestroy; @@ -32,7 +35,9 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -229,4 +234,38 @@ public void registerGlobalSystemConnector(GlobalSystemConnector connector) catalogLoadingLock.unlock(); } } + + public static class NoOpWorkerCatalogManager + implements CatalogManager + { + @Override + public Set getCatalogNames() + { + return Set.of(); + } + + @Override + public Optional getCatalog(CatalogName catalogName) + { + return Optional.empty(); + } + + @Override + public Optional getCatalogProperties(CatalogHandle catalogHandle) + { + return Optional.empty(); + } + + @Override + public Set getActiveCatalogs() + { + return Set.of(); + } + + @Override + public void createCatalog(CatalogName catalogName, ConnectorName connectorName, Map properties, boolean notExists) {} + + @Override + public void dropCatalog(CatalogName catalogName, boolean exists) {} + } } diff --git a/core/trino-main/src/main/java/io/trino/connector/system/CatalogSystemTable.java b/core/trino-main/src/main/java/io/trino/connector/system/CatalogSystemTable.java index 0c6a6f5ef6e8..32278e64b362 100644 --- a/core/trino-main/src/main/java/io/trino/connector/system/CatalogSystemTable.java +++ b/core/trino-main/src/main/java/io/trino/connector/system/CatalogSystemTable.java @@ -29,9 +29,6 @@ import io.trino.spi.connector.SystemTable; import io.trino.spi.predicate.TupleDomain; -import java.util.List; - -import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.metadata.MetadataListing.listCatalogs; import static io.trino.metadata.MetadataUtil.TableMetadataBuilder.tableMetadataBuilder; import static io.trino.spi.connector.SystemTable.Distribution.SINGLE_COORDINATOR; @@ -47,6 +44,7 @@ public class CatalogSystemTable .column("catalog_name", createUnboundedVarcharType()) .column("connector_id", createUnboundedVarcharType()) .column("connector_name", createUnboundedVarcharType()) + .column("state", createUnboundedVarcharType()) .build(); private final Metadata metadata; private final AccessControl accessControl; @@ -75,14 +73,12 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect { Session session = ((FullConnectorSession) connectorSession).getSession(); Builder table = InMemoryRecordSet.builder(CATALOG_TABLE); - List catalogInfos = listCatalogs(session, metadata, accessControl).stream() - .filter(CatalogInfo::loaded) - .collect(toImmutableList()); - for (CatalogInfo catalogInfo : catalogInfos) { + for (CatalogInfo catalogInfo : listCatalogs(session, metadata, accessControl)) { table.addRow( catalogInfo.catalogName(), catalogInfo.catalogName(), - catalogInfo.connectorName().toString()); + catalogInfo.connectorName().toString(), + catalogInfo.catalogStatus().toString()); } return table.build().cursor(); } diff --git a/core/trino-main/src/main/java/io/trino/connector/system/TaskSystemTable.java b/core/trino-main/src/main/java/io/trino/connector/system/TaskSystemTable.java index 34889410a148..88ab3c23a815 100644 --- a/core/trino-main/src/main/java/io/trino/connector/system/TaskSystemTable.java +++ b/core/trino-main/src/main/java/io/trino/connector/system/TaskSystemTable.java @@ -113,8 +113,8 @@ public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, Connect nodeId, taskStatus.getTaskId().toString(), - taskStatus.getTaskId().getStageId().toString(), - taskStatus.getTaskId().getQueryId().toString(), + taskStatus.getTaskId().stageId().toString(), + taskStatus.getTaskId().queryId().toString(), taskStatus.getState().toString(), (long) stats.getTotalDrivers(), diff --git a/core/trino-main/src/main/java/io/trino/cost/CachingCostProvider.java b/core/trino-main/src/main/java/io/trino/cost/CachingCostProvider.java index 7abcc9342f31..cc4900a8cdfa 100644 --- a/core/trino-main/src/main/java/io/trino/cost/CachingCostProvider.java +++ b/core/trino-main/src/main/java/io/trino/cost/CachingCostProvider.java @@ -38,7 +38,7 @@ public class CachingCostProvider private final Optional memo; private final Session session; - private final Map cache = new IdentityHashMap<>(); + private final Map cache = new IdentityHashMap<>(0); public CachingCostProvider(CostCalculator costCalculator, StatsProvider statsProvider, Session session) { diff --git a/core/trino-main/src/main/java/io/trino/cost/CachingStatsProvider.java b/core/trino-main/src/main/java/io/trino/cost/CachingStatsProvider.java index 4120872b3bd1..9ec5bf8bf404 100644 --- a/core/trino-main/src/main/java/io/trino/cost/CachingStatsProvider.java +++ b/core/trino-main/src/main/java/io/trino/cost/CachingStatsProvider.java @@ -42,7 +42,7 @@ public final class CachingStatsProvider private final TableStatsProvider tableStatsProvider; private final RuntimeInfoProvider runtimeInfoProvider; - private final Map cache = new IdentityHashMap<>(); + private final Map cache = new IdentityHashMap<>(0); public CachingStatsProvider(StatsCalculator statsCalculator, Session session, TableStatsProvider tableStatsProvider) { diff --git a/core/trino-main/src/main/java/io/trino/cost/ComposableStatsCalculator.java b/core/trino-main/src/main/java/io/trino/cost/ComposableStatsCalculator.java index ce01afb134e8..04dc9c5c19ec 100644 --- a/core/trino-main/src/main/java/io/trino/cost/ComposableStatsCalculator.java +++ b/core/trino-main/src/main/java/io/trino/cost/ComposableStatsCalculator.java @@ -17,7 +17,7 @@ import com.google.common.collect.ListMultimap; import com.google.inject.Inject; import io.trino.matching.Pattern; -import io.trino.matching.pattern.TypeOfPattern; +import io.trino.matching.TypeOfPattern; import io.trino.sql.planner.plan.PlanNode; import java.lang.reflect.Modifier; diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java b/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java index 3c92cf4cd00a..e0b78689f27c 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java @@ -336,6 +336,7 @@ private static QueryStats immediateFailureQueryStats() DataSize.ofBytes(0), ImmutableList.of(), DynamicFiltersStats.EMPTY, + ImmutableMap.of(), ImmutableList.of(), ImmutableList.of()); } diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java index 082d4a8daa7c..50e919ee97b7 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java @@ -86,6 +86,9 @@ import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.airlift.jaxrs.AsyncResponseHandler.bindAsyncResponse; import static io.trino.client.ProtocolHeaders.TRINO_HEADERS; +import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.ABANDONED; +import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.NOT_SUBMITTED; +import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.SUBMITTED; import static io.trino.execution.QueryState.FAILED; import static io.trino.execution.QueryState.QUEUED; import static io.trino.server.ServletSecurityUtils.authenticatedIdentity; @@ -100,6 +103,7 @@ import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @Path("/v1/statement") @@ -263,7 +267,7 @@ private static URI getQueuedUri(QueryId queryId, Slug slug, long token, External { return externalUriInfo.baseUriBuilder() .path("/v1/statement/queued/") - .path(queryId.toString()) + .path(queryId.id()) .path(slug.makeSlug(QUEUED_QUERY, token)) .path(String.valueOf(token)) .build(); @@ -280,7 +284,7 @@ private static QueryResults createQueryResults( { QueryState state = queryError.map(error -> FAILED).orElse(QUEUED); return new QueryResults( - queryId.toString(), + queryId.id(), getQueryInfoUri(queryInfoUrl, queryId, externalUriInfo), null, nextUri, @@ -300,6 +304,13 @@ private static QueryResults createQueryResults( OptionalLong.empty()); } + enum SubmissionState + { + NOT_SUBMITTED, + SUBMITTED, + ABANDONED + } + private static final class Query { private final String query; @@ -312,7 +323,7 @@ private static final class Query private final AtomicLong lastToken = new AtomicLong(); private final long initTime = System.nanoTime(); - private final AtomicReference submissionGate = new AtomicReference<>(); + private final AtomicReference submissionGate = new AtomicReference<>(NOT_SUBMITTED); private final SettableFuture creationFuture = SettableFuture.create(); public Query(String query, SessionContext sessionContext, DispatchManager dispatchManager, QueryInfoUrlFactory queryInfoUrlFactory, Tracer tracer) @@ -325,7 +336,7 @@ public Query(String query, SessionContext sessionContext, DispatchManager dispat this.queryInfoUrl = queryInfoUrlFactory.getQueryInfoUrl(queryId); requireNonNull(tracer, "tracer is null"); this.querySpan = tracer.spanBuilder("query") - .setAttribute(TrinoAttributes.QUERY_ID, queryId.toString()) + .setAttribute(TrinoAttributes.QUERY_ID, queryId.id()) .startSpan(); } @@ -344,14 +355,14 @@ public long getLastToken() return lastToken.get(); } - public boolean tryAbandonSubmissionWithTimeout(Duration querySubmissionTimeout) + public boolean tryAbandonSubmissionWithTimeout(long querySubmissionTimeoutNanos) { - return Duration.nanosSince(initTime).compareTo(querySubmissionTimeout) >= 0 && submissionGate.compareAndSet(null, false); + return (System.nanoTime() - initTime) >= querySubmissionTimeoutNanos && submissionGate.compareAndSet(NOT_SUBMITTED, ABANDONED); } public boolean isSubmissionAbandoned() { - return Boolean.FALSE.equals(submissionGate.get()); + return ABANDONED.equals(submissionGate.get()); } public boolean isCreated() @@ -371,7 +382,7 @@ private ListenableFuture waitForDispatched() private void submitIfNeeded() { - if (submissionGate.compareAndSet(null, true)) { + if (submissionGate.compareAndSet(NOT_SUBMITTED, SUBMITTED)) { querySpan.addEvent("submit"); creationFuture.setFuture(dispatchManager.createQuery(queryId, querySpan, slug, sessionContext, query)); } @@ -446,7 +457,7 @@ private URI getRedirectUri(CoordinatorLocation coordinatorLocation, ExternalUriI { return coordinatorLocation.getUri(externalUriInfo) .path("/v1/statement/executing") - .path(queryId.toString()) + .path(queryId.id()) .path(slug.makeSlug(EXECUTING_QUERY, 0)) .path("0") .build(); @@ -480,11 +491,11 @@ private static class QueryManager private final ConcurrentMap queries = new ConcurrentHashMap<>(); private final ScheduledExecutorService scheduledExecutorService = newSingleThreadScheduledExecutor(daemonThreadsNamed("drain-state-query-manager")); - private final Duration querySubmissionTimeout; + private final long querySubmissionTimeoutNanos; public QueryManager(Duration querySubmissionTimeout) { - this.querySubmissionTimeout = requireNonNull(querySubmissionTimeout, "querySubmissionTimeout is null"); + this.querySubmissionTimeoutNanos = requireNonNull(querySubmissionTimeout, "querySubmissionTimeout is null").roundTo(NANOSECONDS); } public void initialize(DispatchManager dispatchManager) @@ -520,7 +531,7 @@ private boolean shouldBePurged(DispatchManager dispatchManager, Query query) // Query submission was explicitly abandoned return true; } - if (query.tryAbandonSubmissionWithTimeout(querySubmissionTimeout)) { + if (query.tryAbandonSubmissionWithTimeout(querySubmissionTimeoutNanos)) { // Query took too long to be submitted by the client return true; } diff --git a/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java b/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java index 13b41b9e0626..20b71e3e13c8 100644 --- a/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java +++ b/core/trino-main/src/main/java/io/trino/event/QueryMonitor.java @@ -215,6 +215,7 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), 0, 0, 0, @@ -241,6 +242,7 @@ public void queryImmediateFailureEvent(BasicQueryInfo queryInfo, ExecutionFailur ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), + ImmutableMap.of(), Optional.empty()), createQueryContext( queryInfo.getSession(), @@ -331,6 +333,7 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo) Optional.of(queryStats.getOutputBlockedTime().toJavaTime()), Optional.of(queryStats.getFailedOutputBlockedTime().toJavaTime()), Optional.of(queryStats.getPhysicalInputReadTime().toJavaTime()), + Optional.of(queryStats.getFinishingTime().toJavaTime()), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTaskUserMemory().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), @@ -357,6 +360,7 @@ private QueryStatistics createQueryStatistics(QueryInfo queryInfo) getDynamicFilterDomainStats(queryInfo), memoize(() -> operatorStats.stream().map(operatorStatsCodec::toJson).toList()), ImmutableList.copyOf(queryInfo.getQueryStats().getOptimizerRulesSummaries()), + ImmutableMap.copyOf(queryInfo.getQueryStats().getCatalogMetadataMetrics()), serializedPlanNodeStatsAndCosts); } @@ -569,7 +573,7 @@ private Optional createQueryFailureInfo(ExecutionFailureInfo f private static Optional findFailedTask(StagesInfo stages) { - for (StageInfo stageInfo : stages.getSubStagesDeepPostOrder(stages.getOutputStageId(), true)) { + for (StageInfo stageInfo : stages.getSubStagesDeep(stages.getOutputStageId(), true)) { Optional failedTaskInfo = stageInfo.getTasks().stream() .filter(taskInfo -> taskInfo.taskStatus().getState() == TaskState.FAILED) .findFirst(); @@ -732,7 +736,7 @@ private static StageCpuDistribution computeCpuDistribution(StageInfo stageInfo) DistributionSnapshot snapshot = cpuDistribution.snapshot(); return new StageCpuDistribution( - stageInfo.getStageId().getId(), + stageInfo.getStageId().id(), stageInfo.getTasks().size(), (long) snapshot.p25(), (long) snapshot.p50(), @@ -759,7 +763,7 @@ private static Optional computeStageOutputBufferUt { return stageInfo.getStageStats().getOutputBufferUtilization() .map(utilization -> new StageOutputBufferUtilization( - stageInfo.getStageId().getId(), + stageInfo.getStageId().id(), stageInfo.getTasks().size(), // scale ratio to percentages utilization.p01() * 100, @@ -791,7 +795,7 @@ private static Optional computeStageOutputBufferMetric if (metrics.getMetrics().isEmpty()) { return Optional.empty(); } - return Optional.of(new StageOutputBufferMetrics(stageInfo.getStageId().getId(), metrics)); + return Optional.of(new StageOutputBufferMetrics(stageInfo.getStageId().id(), metrics)); } private List getStageTaskStatistics(QueryInfo queryInfo) @@ -855,7 +859,7 @@ private StageTaskStatistics computeStageTaskStatistics(QueryInfo queryInfo, Stag distributionSnapshot.count()); })); return new StageTaskStatistics( - stageInfo.getStageId().getId(), + stageInfo.getStageId().id(), stageInfo.getTasks().size(), getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getTotalCpuTime().toMillis())), getTasksDistribution(stageInfo, taskInfo -> Optional.of(taskInfo.stats().getTotalScheduledTime().toMillis())), diff --git a/core/trino-main/src/main/java/io/trino/execution/CreateCatalogTask.java b/core/trino-main/src/main/java/io/trino/execution/CreateCatalogTask.java index 2a9ad0fc4b27..55d38352e14a 100644 --- a/core/trino-main/src/main/java/io/trino/execution/CreateCatalogTask.java +++ b/core/trino-main/src/main/java/io/trino/execution/CreateCatalogTask.java @@ -17,7 +17,6 @@ import com.google.inject.Inject; import io.trino.Session; import io.trino.execution.warnings.WarningCollector; -import io.trino.metadata.CatalogManager; import io.trino.security.AccessControl; import io.trino.spi.TrinoException; import io.trino.spi.catalog.CatalogName; @@ -47,14 +46,12 @@ public class CreateCatalogTask { private final PlannerContext plannerContext; private final AccessControl accessControl; - private final CatalogManager catalogManager; @Inject - public CreateCatalogTask(PlannerContext plannerContext, AccessControl accessControl, CatalogManager catalogManager) + public CreateCatalogTask(PlannerContext plannerContext, AccessControl accessControl) { this.plannerContext = requireNonNull(plannerContext, "plannerContext is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); - this.catalogManager = requireNonNull(catalogManager, "catalogManager is null"); } @Override @@ -99,7 +96,7 @@ public ListenableFuture execute( } ConnectorName connectorName = new ConnectorName(statement.getConnectorName().toString()); - catalogManager.createCatalog(catalog, connectorName, properties, statement.isNotExists()); + plannerContext.getMetadata().createCatalog(session, catalog, connectorName, properties, statement.isNotExists()); return immediateVoidFuture(); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/DropCatalogTask.java b/core/trino-main/src/main/java/io/trino/execution/DropCatalogTask.java index 58954a534048..a7b72e1c6020 100644 --- a/core/trino-main/src/main/java/io/trino/execution/DropCatalogTask.java +++ b/core/trino-main/src/main/java/io/trino/execution/DropCatalogTask.java @@ -17,7 +17,7 @@ import com.google.inject.Inject; import io.trino.connector.system.GlobalSystemConnector; import io.trino.execution.warnings.WarningCollector; -import io.trino.metadata.CatalogManager; +import io.trino.metadata.Metadata; import io.trino.security.AccessControl; import io.trino.spi.TrinoException; import io.trino.spi.catalog.CatalogName; @@ -34,13 +34,13 @@ public class DropCatalogTask implements DataDefinitionTask { - private final CatalogManager catalogManager; + private final Metadata metadata; private final AccessControl accessControl; @Inject - public DropCatalogTask(CatalogManager catalogManager, AccessControl accessControl) + public DropCatalogTask(Metadata metadata, AccessControl accessControl) { - this.catalogManager = requireNonNull(catalogManager, "catalogManager is null"); + this.metadata = requireNonNull(metadata, "metadata is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); } @@ -66,7 +66,7 @@ public ListenableFuture execute( throw new TrinoException(NOT_SUPPORTED, "Dropping system catalog is not allowed"); } accessControl.checkCanDropCatalog(stateMachine.getSession().toSecurityContext(), catalogName); - catalogManager.dropCatalog(new CatalogName(catalogName), statement.isExists()); + metadata.dropCatalog(stateMachine.getSession(), new CatalogName(catalogName), statement.isExists()); return immediateVoidFuture(); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryInfo.java b/core/trino-main/src/main/java/io/trino/execution/QueryInfo.java index 0b8f07cdeef8..efd508c6582b 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryInfo.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryInfo.java @@ -448,45 +448,4 @@ public String toString() .add("fieldNames", fieldNames) .toString(); } - - public QueryInfo pruneDigests() - { - return new QueryInfo( - queryId, - session, - state, - self, - fieldNames, - query, - preparedQuery, - queryStats, - setCatalog, - setSchema, - setPath, - setAuthorizationUser, - resetAuthorizationUser, - setOriginalRoles, - setSessionProperties, - resetSessionProperties, - setRoles, - addedPreparedStatements, - deallocatedPreparedStatements, - startedTransactionId, - clearTransactionId, - updateType, - stages.map(StagesInfo::pruneDigests), - failureInfo, - errorCode, - warnings, - inputs, - output, - referencedTables, - routines, - finalQueryInfo, - resourceGroupId, - queryType, - retryPolicy, - pruned, - version); - } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java index 911310064dbe..0d057c21b9f6 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java @@ -13,6 +13,7 @@ */ package io.trino.execution; +import com.google.common.collect.ImmutableSet; import io.airlift.configuration.Config; import io.airlift.configuration.ConfigDescription; import io.airlift.configuration.DefunctConfig; @@ -22,12 +23,15 @@ import io.airlift.units.MinDataSize; import io.airlift.units.MinDuration; import io.trino.operator.RetryPolicy; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.DecimalMin; import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; +import java.util.EnumSet; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import static com.google.common.base.Preconditions.checkArgument; @@ -104,6 +108,8 @@ public class QueryManagerConfig private Duration requiredWorkersMaxWait = new Duration(5, TimeUnit.MINUTES); private RetryPolicy retryPolicy = RetryPolicy.NONE; + private Set allowedRetryPolicies = EnumSet.allOf(RetryPolicy.class); + private int queryRetryAttempts = 4; private int taskRetryAttemptsPerTask = 4; private Duration retryInitialDelay = new Duration(10, SECONDS); @@ -613,6 +619,25 @@ public QueryManagerConfig setRetryPolicy(RetryPolicy retryPolicy) return this; } + public Set getAllowedRetryPolicies() + { + return allowedRetryPolicies; + } + + @Config("retry-policy.allowed") + @ConfigDescription("Retry policies that are allowed to be used") + public QueryManagerConfig setAllowedRetryPolicies(Set allowedRetryPolicies) + { + this.allowedRetryPolicies = ImmutableSet.copyOf(allowedRetryPolicies); + return this; + } + + @AssertTrue(message = "Selected retry policy not present in retry-policy.allowed list") + public boolean isRetryPolicyAllowed() + { + return allowedRetryPolicies.contains(retryPolicy); + } + @Min(0) public int getQueryRetryAttempts() { diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryManagerStats.java b/core/trino-main/src/main/java/io/trino/execution/QueryManagerStats.java index e3ed0a2ece57..592fcf17e520 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryManagerStats.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryManagerStats.java @@ -24,8 +24,8 @@ import org.weakref.jmx.Nested; import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; import java.util.function.Supplier; import static io.trino.execution.QueryState.RUNNING; @@ -305,23 +305,24 @@ public long getQueuedDrivers() public void updateDriverStats(QueryTracker queryTracker) { - AtomicInteger tmpQueuedDrivers = new AtomicInteger(); - AtomicInteger tmpRunningDrivers = new AtomicInteger(); - AtomicInteger tmpBlockedDrivers = new AtomicInteger(); - AtomicInteger tmpCompletedDrivers = new AtomicInteger(); + LongAdder tmpQueuedDrivers = new LongAdder(); + LongAdder tmpRunningDrivers = new LongAdder(); + LongAdder tmpBlockedDrivers = new LongAdder(); + LongAdder tmpCompletedDrivers = new LongAdder(); queryTracker.getAllQueries().stream() .filter(query -> !query.getState().isDone()) .map(dispatchQuery -> dispatchQuery.getBasicQueryInfo().getQueryStats()) + .parallel() .forEach(stats -> { - tmpQueuedDrivers.addAndGet(stats.getQueuedDrivers()); - tmpRunningDrivers.addAndGet(stats.getRunningDrivers()); - tmpBlockedDrivers.addAndGet(stats.getBlockedDrivers()); - tmpCompletedDrivers.addAndGet(stats.getCompletedDrivers()); + tmpQueuedDrivers.add(stats.getQueuedDrivers()); + tmpRunningDrivers.add(stats.getRunningDrivers()); + tmpBlockedDrivers.add(stats.getBlockedDrivers()); + tmpCompletedDrivers.add(stats.getCompletedDrivers()); }); - queuedDrivers.set(tmpQueuedDrivers.get()); - runningDrivers.set(tmpRunningDrivers.get()); - blockedDrivers.set(tmpBlockedDrivers.get()); - completedDrivers.set(tmpCompletedDrivers.get()); + queuedDrivers.set(tmpQueuedDrivers.sum()); + runningDrivers.set(tmpRunningDrivers.sum()); + blockedDrivers.set(tmpBlockedDrivers.sum()); + completedDrivers.set(tmpCompletedDrivers.sum()); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java index 93a88804ec0d..634f4c39de92 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java @@ -29,6 +29,7 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.StatusCode; +import io.trino.NotInTransactionException; import io.trino.Session; import io.trino.client.NodeVersion; import io.trino.exchange.ExchangeInput; @@ -36,6 +37,7 @@ import io.trino.execution.StateMachine.StateChangeListener; import io.trino.execution.querystats.PlanOptimizersStatsCollector; import io.trino.execution.warnings.WarningCollector; +import io.trino.metadata.CatalogInfo; import io.trino.metadata.Metadata; import io.trino.operator.BlockedReason; import io.trino.operator.OperatorStats; @@ -182,6 +184,7 @@ public class QueryStateMachine private final AtomicReference> output = new AtomicReference<>(Optional.empty()); private final AtomicReference> referencedTables = new AtomicReference<>(ImmutableList.of()); private final AtomicReference> routines = new AtomicReference<>(ImmutableList.of()); + private final AtomicReference> catalogMetadataMetrics = new AtomicReference<>(ImmutableMap.of()); private final StateMachine> finalQueryInfo; private final WarningCollector warningCollector; @@ -386,6 +389,38 @@ static QueryStateMachine beginWithTicker( return queryStateMachine; } + private void collectCatalogMetadataMetrics() + { + try { + if (session.getTransactionId().filter(transactionManager::transactionExists).isEmpty()) { + // The metrics collection depends on active transaction as the metrics + // are stored in the transactional ConnectorMetadata, but the collection can be + // run after the query has failed e.g., via cancel. + return; + } + + ImmutableMap.Builder catalogMetadataMetrics = ImmutableMap.builder(); + List activeCatalogs = metadata.listActiveCatalogs(session); + for (CatalogInfo activeCatalog : activeCatalogs) { + Metrics metrics = metadata.getMetrics(session, activeCatalog.catalogName()); + if (!metrics.getMetrics().isEmpty()) { + catalogMetadataMetrics.put(activeCatalog.catalogName(), metrics); + } + } + + this.catalogMetadataMetrics.set(catalogMetadataMetrics.buildOrThrow()); + } + catch (NotInTransactionException e) { + // Ignore, The metrics collection depends on an active transaction and should be skipped + // if there is no one running. + // The exception can be thrown even though there is a check for transaction at the top of the method, because + // the transaction can be committed or aborted concurrently, after the check is done. + } + catch (RuntimeException e) { + QUERY_STATE_LOG.error(e, "Error collecting query catalog metadata metrics: %s", queryId); + } + } + public QueryId getQueryId() { return queryId; @@ -449,15 +484,27 @@ public void updateMemoryUsage( long taskRevocableMemoryInBytes, long taskTotalMemoryInBytes) { - currentUserMemory.addAndGet(deltaUserMemoryInBytes); - currentRevocableMemory.addAndGet(deltaRevocableMemoryInBytes); - currentTotalMemory.addAndGet(deltaTotalMemoryInBytes); - peakUserMemory.updateAndGet(currentPeakValue -> Math.max(currentUserMemory.get(), currentPeakValue)); - peakRevocableMemory.updateAndGet(currentPeakValue -> Math.max(currentRevocableMemory.get(), currentPeakValue)); - peakTotalMemory.updateAndGet(currentPeakValue -> Math.max(currentTotalMemory.get(), currentPeakValue)); - peakTaskUserMemory.accumulateAndGet(taskUserMemoryInBytes, Math::max); - peakTaskRevocableMemory.accumulateAndGet(taskRevocableMemoryInBytes, Math::max); - peakTaskTotalMemory.accumulateAndGet(taskTotalMemoryInBytes, Math::max); + long currentUserMemory = this.currentUserMemory.addAndGet(deltaUserMemoryInBytes); + long currentRevocableMemory = this.currentRevocableMemory.addAndGet(deltaRevocableMemoryInBytes); + long currentTotalMemory = this.currentTotalMemory.addAndGet(deltaTotalMemoryInBytes); + if (currentUserMemory > peakUserMemory.get()) { + peakUserMemory.accumulateAndGet(currentUserMemory, Math::max); + } + if (currentRevocableMemory > peakRevocableMemory.get()) { + peakRevocableMemory.accumulateAndGet(currentRevocableMemory, Math::max); + } + if (currentTotalMemory > peakTotalMemory.get()) { + peakTotalMemory.accumulateAndGet(currentTotalMemory, Math::max); + } + if (taskUserMemoryInBytes > peakTaskUserMemory.get()) { + peakTaskUserMemory.accumulateAndGet(taskUserMemoryInBytes, Math::max); + } + if (taskRevocableMemoryInBytes > peakTaskRevocableMemory.get()) { + peakTaskRevocableMemory.accumulateAndGet(taskRevocableMemoryInBytes, Math::max); + } + if (taskTotalMemoryInBytes > peakTaskTotalMemory.get()) { + peakTaskTotalMemory.accumulateAndGet(taskTotalMemoryInBytes, Math::max); + } } public BasicQueryInfo getBasicQueryInfo(Optional rootStage) @@ -851,6 +898,10 @@ private QueryStats getQueryStats(Optional stages) } } + // Try to collect the catalog metadata metrics. + // The collection will fail, and we will use metrics collected earlier if + // the query is already committed or aborted and metadata is not available. + collectCatalogMetadataMetrics(); return new QueryStats( queryStateTimer.getCreateTime(), getExecutionStartTime().orElse(null), @@ -936,7 +987,7 @@ private QueryStats getQueryStats(Optional stages) stageGcStatistics.build(), getDynamicFiltersStats(), - + catalogMetadataMetrics.get(), operatorStatsSummary.build(), planOptimizersStatsCollector.getTopRuleStats()); } @@ -1167,6 +1218,7 @@ public boolean transitionToFinishing() transitionToFailed(e); return true; } + collectCatalogMetadataMetrics(); Optional transaction = session.getTransactionId().flatMap(transactionManager::getTransactionInfoIfExist); if (transaction.isPresent() && transaction.get().isAutoCommitContext()) { @@ -1244,6 +1296,7 @@ private boolean transitionToFailed(Throwable throwable, boolean log) } return false; } + collectCatalogMetadataMetrics(); try { if (log) { @@ -1549,6 +1602,7 @@ private static QueryStats pruneQueryStats(QueryStats queryStats) queryStats.getFailedPhysicalWrittenDataSize(), queryStats.getStageGcStatistics(), queryStats.getDynamicFiltersStats(), + ImmutableMap.of(), ImmutableList.of(), // Remove the operator summaries as OperatorInfo (especially DirectExchangeClientStatus) can hold onto a large amount of memory ImmutableList.of()); } diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryStateTimer.java b/core/trino-main/src/main/java/io/trino/execution/QueryStateTimer.java index 8ca5a8d03129..4f2176728b95 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryStateTimer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryStateTimer.java @@ -26,7 +26,6 @@ import static java.lang.Math.max; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.concurrent.TimeUnit.NANOSECONDS; class QueryStateTimer { @@ -321,7 +320,7 @@ private Optional toInstant(AtomicReference instantNanos) private Instant toInstant(long instantNanos) { - return createTime.plusMillis(NANOSECONDS.toMillis(instantNanos - createNanos)); + return createTime.plusNanos(instantNanos - createNanos); } private static long currentThreadCpuTime() diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryStats.java b/core/trino-main/src/main/java/io/trino/execution/QueryStats.java index c940dd064921..370e35978c1d 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryStats.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryStats.java @@ -24,15 +24,16 @@ import io.trino.operator.TableWriterOperator; import io.trino.spi.eventlistener.QueryPlanOptimizerStatistics; import io.trino.spi.eventlistener.StageGcStatistics; +import io.trino.spi.metrics.Metrics; import jakarta.annotation.Nullable; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.OptionalDouble; import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.units.DataSize.succinctBytes; import static io.trino.server.DynamicFilterService.DynamicFiltersStats; import static java.util.Objects.requireNonNull; @@ -127,6 +128,7 @@ public class QueryStats private final DynamicFiltersStats dynamicFiltersStats; + private final Map catalogMetadataMetrics; private final List operatorSummaries; private final List optimizerRulesSummaries; @@ -218,7 +220,7 @@ public QueryStats( @JsonProperty("stageGcStatistics") List stageGcStatistics, @JsonProperty("dynamicFiltersStats") DynamicFiltersStats dynamicFiltersStats, - + @JsonProperty("catalogMetadataMetrics") Map catalogMetadataMetrics, @JsonProperty("operatorSummaries") List operatorSummaries, @JsonProperty("optimizerRulesSummaries") List optimizerRulesSummaries) { @@ -323,10 +325,8 @@ public QueryStats( this.stageGcStatistics = ImmutableList.copyOf(requireNonNull(stageGcStatistics, "stageGcStatistics is null")); this.dynamicFiltersStats = requireNonNull(dynamicFiltersStats, "dynamicFiltersStats is null"); - - requireNonNull(operatorSummaries, "operatorSummaries is null"); - this.operatorSummaries = operatorSummaries.stream().map(OperatorStats::pruneDigests).collect(toImmutableList()); - + this.catalogMetadataMetrics = requireNonNull(catalogMetadataMetrics, "catalogMetadataMetrics is null"); + this.operatorSummaries = ImmutableList.copyOf(operatorSummaries); this.optimizerRulesSummaries = ImmutableList.copyOf(requireNonNull(optimizerRulesSummaries, "optimizerRulesSummaries is null")); } @@ -770,6 +770,12 @@ public DynamicFiltersStats getDynamicFiltersStats() return dynamicFiltersStats; } + @JsonProperty + public Map getCatalogMetadataMetrics() + { + return catalogMetadataMetrics; + } + @JsonProperty public List getOperatorSummaries() { diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryTracker.java b/core/trino-main/src/main/java/io/trino/execution/QueryTracker.java index 03107bde993c..5ab430596b98 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryTracker.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryTracker.java @@ -146,7 +146,7 @@ public T getQuery(QueryId queryId) throws NoSuchElementException { return tryGetQuery(queryId) - .orElseThrow(() -> new NoSuchElementException(queryId.toString())); + .orElseThrow(() -> new NoSuchElementException(queryId.id())); } public boolean hasQuery(QueryId queryId) diff --git a/core/trino-main/src/main/java/io/trino/execution/SqlQueryManager.java b/core/trino-main/src/main/java/io/trino/execution/SqlQueryManager.java index 41914812a553..745c485d8241 100644 --- a/core/trino-main/src/main/java/io/trino/execution/SqlQueryManager.java +++ b/core/trino-main/src/main/java/io/trino/execution/SqlQueryManager.java @@ -176,7 +176,7 @@ public void setOutputInfoListener(QueryId queryId, Consumer lis @Override public void outputTaskFailed(TaskId taskId, Throwable failure) { - queryTracker.getQuery(taskId.getQueryId()).outputTaskFailed(taskId, failure); + queryTracker.getQuery(taskId.queryId()).outputTaskFailed(taskId, failure); } @Override @@ -311,7 +311,7 @@ public void cancelStage(StageId stageId) log.debug("Cancel stage %s", stageId); - queryTracker.tryGetQuery(stageId.getQueryId()) + queryTracker.tryGetQuery(stageId.queryId()) .ifPresent(query -> query.cancelStage(stageId)); } diff --git a/core/trino-main/src/main/java/io/trino/execution/SqlTask.java b/core/trino-main/src/main/java/io/trino/execution/SqlTask.java index f91da1f51c7b..c85c0bf3fa0e 100644 --- a/core/trino-main/src/main/java/io/trino/execution/SqlTask.java +++ b/core/trino-main/src/main/java/io/trino/execution/SqlTask.java @@ -252,6 +252,12 @@ public Instant getTaskCreatedTime() return taskStateMachine.getCreatedTime(); } + @Nullable + public Instant getTaskEndTime() + { + return taskStateMachine.getEndTime(); + } + public TaskId getTaskId() { return taskStateMachine.getTaskId(); @@ -267,6 +273,11 @@ public void recordHeartbeat() lastHeartbeat.set(Instant.now()); } + public Instant lastHeartbeat() + { + return lastHeartbeat.get(); + } + public TaskInfo getTaskInfo() { try (SetThreadName _ = new SetThreadName("Task-" + taskId)) { @@ -363,10 +374,10 @@ else if (taskHolder.getTaskExecution() != null) { TaskContext taskContext = taskHolder.getTaskExecution().getTaskContext(); for (PipelineContext pipelineContext : taskContext.getPipelineContexts()) { PipelineStatus pipelineStatus = pipelineContext.getPipelineStatus(); - queuedPartitionedDrivers += pipelineStatus.getQueuedPartitionedDrivers(); - queuedPartitionedSplitsWeight += pipelineStatus.getQueuedPartitionedSplitsWeight(); - runningPartitionedDrivers += pipelineStatus.getRunningPartitionedDrivers(); - runningPartitionedSplitsWeight += pipelineStatus.getRunningPartitionedSplitsWeight(); + queuedPartitionedDrivers += pipelineStatus.queuedPartitionedDrivers(); + queuedPartitionedSplitsWeight += pipelineStatus.queuedPartitionedSplitsWeight(); + runningPartitionedDrivers += pipelineStatus.runningPartitionedDrivers(); + runningPartitionedSplitsWeight += pipelineStatus.runningPartitionedSplitsWeight(); physicalWrittenBytes += pipelineContext.getPhysicalWrittenDataSize(); } writerInputDataSize = succinctBytes(taskContext.getWriterInputDataSize()); @@ -555,8 +566,8 @@ private SqlTaskExecution tryCreateSqlTaskExecution(Session session, Span stageSp taskSpan.set(tracer.spanBuilder("task") .setParent(Context.current().with(stageSpan)) - .setAttribute(TrinoAttributes.QUERY_ID, taskId.getQueryId().toString()) - .setAttribute(TrinoAttributes.STAGE_ID, taskId.getStageId().toString()) + .setAttribute(TrinoAttributes.QUERY_ID, taskId.queryId().toString()) + .setAttribute(TrinoAttributes.STAGE_ID, taskId.stageId().toString()) .setAttribute(TrinoAttributes.TASK_ID, taskId.toString()) .startSpan()); diff --git a/core/trino-main/src/main/java/io/trino/execution/SqlTaskExecution.java b/core/trino-main/src/main/java/io/trino/execution/SqlTaskExecution.java index cce6527dccd3..bf5a1f84c6dd 100644 --- a/core/trino-main/src/main/java/io/trino/execution/SqlTaskExecution.java +++ b/core/trino-main/src/main/java/io/trino/execution/SqlTaskExecution.java @@ -595,10 +595,10 @@ private DriverSplitRunnerFactory(DriverFactory driverFactory, Tracer tracer, boo this.pipelineContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver(), partitioned); this.pipelineSpan = tracer.spanBuilder("pipeline") .setParent(Context.current().with(taskSpan)) - .setAttribute(TrinoAttributes.QUERY_ID, taskId.getQueryId().toString()) - .setAttribute(TrinoAttributes.STAGE_ID, taskId.getStageId().toString()) + .setAttribute(TrinoAttributes.QUERY_ID, taskId.queryId().toString()) + .setAttribute(TrinoAttributes.STAGE_ID, taskId.stageId().toString()) .setAttribute(TrinoAttributes.TASK_ID, taskId.toString()) - .setAttribute(TrinoAttributes.PIPELINE_ID, taskId.getStageId() + "-" + pipelineContext.getPipelineId()) + .setAttribute(TrinoAttributes.PIPELINE_ID, taskId.stageId() + "-" + pipelineContext.getPipelineId()) .startSpan(); } diff --git a/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java b/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java index 7cde396ac676..79fe64df4b72 100644 --- a/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java +++ b/core/trino-main/src/main/java/io/trino/execution/SqlTaskManager.java @@ -69,7 +69,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -233,7 +232,7 @@ public SqlTaskManager( taskId, locationFactory.createLocalTaskLocation(taskId), nodeInfo.getNodeId(), - queryContexts.getUnchecked(taskId.getQueryId()), + queryContexts.getUnchecked(taskId.queryId()), tracer, sqlTaskExecutionFactory, taskNotificationExecutor, @@ -656,13 +655,11 @@ public TaskInfo failTask(TaskId taskId, Throwable failure) void removeOldTasks() { Instant oldestAllowedTask = Instant.now().minusMillis(infoCacheTime.toMillis()); - tasks.asMap().values().stream() - .map(SqlTask::getTaskInfo) - .filter(Objects::nonNull) - .forEach(taskInfo -> { - TaskId taskId = taskInfo.taskStatus().getTaskId(); + tasks.asMap().values() + .forEach(sqlTask -> { + TaskId taskId = sqlTask.getTaskId(); try { - Instant endTime = taskInfo.stats().getEndTime(); + Instant endTime = sqlTask.getTaskEndTime(); if (endTime != null && endTime.isBefore(oldestAllowedTask)) { // The removal here is concurrency safe with respect to any concurrent loads: the cache has no expiration, // the taskId is in the cache, so there mustn't be an ongoing load. @@ -680,20 +677,20 @@ private void failAbandonedTasks() Instant now = Instant.now(); Instant oldestAllowedHeartbeat = now.minusMillis(clientTimeout.toMillis()); for (SqlTask sqlTask : tasks.asMap().values()) { + TaskId taskId = sqlTask.getTaskId(); try { - TaskInfo taskInfo = sqlTask.getTaskInfo(); - TaskStatus taskStatus = taskInfo.taskStatus(); - if (taskStatus.getState().isDone()) { + TaskState taskState = sqlTask.getTaskState(); + if (taskState.isDone()) { continue; } - Instant lastHeartbeat = taskInfo.lastHeartbeat(); + Instant lastHeartbeat = sqlTask.lastHeartbeat(); if (lastHeartbeat != null && lastHeartbeat.isBefore(oldestAllowedHeartbeat)) { - log.info("Failing abandoned task %s", taskStatus.getTaskId()); - sqlTask.failed(new TrinoException(ABANDONED_TASK, format("Task %s has not been accessed since %s: currentTime %s", taskStatus.getTaskId(), lastHeartbeat, now))); + log.info("Failing abandoned task %s", taskId); + sqlTask.failed(new TrinoException(ABANDONED_TASK, format("Task %s has not been accessed since %s: currentTime %s", taskId, lastHeartbeat, now))); } } catch (RuntimeException e) { - log.warn(e, "Error while inspecting age of task %s", sqlTask.getTaskId()); + log.warn(e, "Error while inspecting age of task %s", taskId); } } } @@ -793,7 +790,7 @@ private Optional createStuckSplitTasksInterrupter( *
  • We find long-running splits; we get A, B, C.
    • *
  • None of those is actually running JONI code.
    • *
  • just before when we investigate stack trace for A, the underlying thread already switched to some other unrelated split D; and D is actually running JONI
    • - * we get the stacktrace for what we believe is A, but it is for D, and we decide we should kill the task that A belongs to + *
  • we get the stacktrace for what we believe is A, but it is for D, and we decide we should kill the task that A belongs to
    • *
  • (clash!!!) wrong decision is made
    • * * A proposed fix and more details of this issue are at: pull/13272. diff --git a/core/trino-main/src/main/java/io/trino/execution/StageId.java b/core/trino-main/src/main/java/io/trino/execution/StageId.java index 0651e03bf145..b055efaf3994 100644 --- a/core/trino-main/src/main/java/io/trino/execution/StageId.java +++ b/core/trino-main/src/main/java/io/trino/execution/StageId.java @@ -19,14 +19,16 @@ import io.trino.sql.planner.plan.PlanFragmentId; import java.util.List; -import java.util.Objects; import static com.google.common.base.Preconditions.checkArgument; +import static io.airlift.slice.SizeOf.instanceSize; import static java.lang.Integer.parseInt; import static java.util.Objects.requireNonNull; -public class StageId +public record StageId(QueryId queryId, int id) { + private static final int INSTANCE_SIZE = instanceSize(StageId.class); + @JsonCreator public static StageId valueOf(String stageId) { @@ -37,7 +39,7 @@ public static StageId valueOf(String stageId) public static StageId valueOf(List ids) { checkArgument(ids.size() == 2, "Expected two ids but got: %s", ids); - return new StageId(new QueryId(ids.get(0)), Integer.parseInt(ids.get(1))); + return new StageId(new QueryId(ids.get(0)), parseInt(ids.get(1))); } public static StageId create(QueryId queryId, PlanFragmentId fragmentId) @@ -45,29 +47,15 @@ public static StageId create(QueryId queryId, PlanFragmentId fragmentId) return new StageId(queryId, parseInt(fragmentId.toString())); } - private final QueryId queryId; - private final int id; - public StageId(String queryId, int id) { this(new QueryId(queryId), id); } - public StageId(QueryId queryId, int id) + public StageId { - this.queryId = requireNonNull(queryId, "queryId is null"); + requireNonNull(queryId, "queryId is null"); checkArgument(id >= 0, "id is negative: %s", id); - this.id = id; - } - - public QueryId getQueryId() - { - return queryId; - } - - public int getId() - { - return id; } @Override @@ -77,23 +65,8 @@ public String toString() return queryId + "." + id; } - @Override - public int hashCode() - { - return Objects.hash(id, queryId); - } - - @Override - public boolean equals(Object obj) + public long getRetainedSizeInBytes() { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - StageId other = (StageId) obj; - return this.id == other.id && - Objects.equals(this.queryId, other.queryId); + return INSTANCE_SIZE + queryId.getRetainedSizeInBytes(); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/StageInfo.java b/core/trino-main/src/main/java/io/trino/execution/StageInfo.java index ea2f3e540bb8..5aab4fb8b46f 100644 --- a/core/trino-main/src/main/java/io/trino/execution/StageInfo.java +++ b/core/trino-main/src/main/java/io/trino/execution/StageInfo.java @@ -28,7 +28,6 @@ import java.util.Map; import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; @Immutable @@ -167,21 +166,6 @@ public StageInfo withSubStages(List subStages) failureCause); } - public StageInfo pruneDigests() - { - return new StageInfo( - stageId, - state, - plan, - coordinatorOnly, - types, - stageStats.pruneDigests(), - tasks.stream().map(TaskInfo::pruneDigests).collect(toImmutableList()), - subStages, - tables, - failureCause); - } - public static StageInfo createInitial(QueryId queryId, StageState state, PlanFragment fragment) { return new StageInfo( diff --git a/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java b/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java index 39d6d55fbc4a..4210e13c8095 100644 --- a/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java +++ b/core/trino-main/src/main/java/io/trino/execution/StageStateMachine.java @@ -29,6 +29,7 @@ import io.trino.operator.OperatorStats; import io.trino.operator.PipelineStats; import io.trino.operator.TaskStats; +import io.trino.plugin.base.metrics.DistributionSnapshot; import io.trino.plugin.base.metrics.TDigestHistogram; import io.trino.spi.eventlistener.StageGcStatistics; import io.trino.spi.metrics.Metrics; @@ -121,7 +122,7 @@ public StageStateMachine( stageSpan = tracer.spanBuilder("stage") .setParent(Context.current().with(schedulerSpan)) - .setAttribute(TrinoAttributes.QUERY_ID, stageId.getQueryId().toString()) + .setAttribute(TrinoAttributes.QUERY_ID, stageId.queryId().toString()) .setAttribute(TrinoAttributes.STAGE_ID, stageId.toString()) .startSpan(); @@ -243,11 +244,15 @@ public long getTotalMemoryReservation() public void updateMemoryUsage(long deltaUserMemoryInBytes, long deltaRevocableMemoryInBytes, long deltaTotalMemoryInBytes) { - currentUserMemory.addAndGet(deltaUserMemoryInBytes); - currentRevocableMemory.addAndGet(deltaRevocableMemoryInBytes); + long currentUserMemory = this.currentUserMemory.addAndGet(deltaUserMemoryInBytes); + long currentRevocableMemory = this.currentRevocableMemory.addAndGet(deltaRevocableMemoryInBytes); currentTotalMemory.addAndGet(deltaTotalMemoryInBytes); - peakUserMemory.updateAndGet(currentPeakValue -> max(currentUserMemory.get(), currentPeakValue)); - peakRevocableMemory.updateAndGet(currentPeakValue -> max(currentRevocableMemory.get(), currentPeakValue)); + if (currentUserMemory > peakUserMemory.get()) { + peakUserMemory.accumulateAndGet(currentUserMemory, Math::max); + } + if (currentRevocableMemory > peakRevocableMemory.get()) { + peakRevocableMemory.accumulateAndGet(currentRevocableMemory, Math::max); + } } public BasicStageStats getBasicStageStats(Supplier> taskInfosSupplier) @@ -643,7 +648,7 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier) succinctDuration(inputBlockedTime, NANOSECONDS), succinctDuration(failedInputBlockedTime, NANOSECONDS), succinctBytes(bufferedDataSize), - TDigestHistogram.merge(bufferUtilizationHistograms.build()).map(DistributionSnapshot::new), + TDigestHistogram.merge(bufferUtilizationHistograms.build()).map(DistributionSnapshot::fromDistribution), succinctBytes(outputDataSize), succinctBytes(failedOutputDataSize), outputPositions, @@ -655,7 +660,7 @@ public StageInfo getStageInfo(Supplier> taskInfosSupplier) succinctBytes(failedPhysicalWrittenDataSize), new StageGcStatistics( - stageId.getId(), + stageId.id(), totalTasks, fullGcTaskCount, minFullGcSec, diff --git a/core/trino-main/src/main/java/io/trino/execution/StageStats.java b/core/trino-main/src/main/java/io/trino/execution/StageStats.java index 21b63673a991..50030b0a301f 100644 --- a/core/trino-main/src/main/java/io/trino/execution/StageStats.java +++ b/core/trino-main/src/main/java/io/trino/execution/StageStats.java @@ -36,9 +36,7 @@ import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.units.DataSize.Unit.BYTE; -import static io.trino.execution.DistributionSnapshot.pruneMetrics; import static io.trino.execution.StageState.RUNNING; import static java.lang.Math.min; import static java.util.Objects.requireNonNull; @@ -102,7 +100,7 @@ public class StageStats private final Duration failedInputBlockedTime; private final DataSize bufferedDataSize; - private final Optional outputBufferUtilization; + private final Optional outputBufferUtilization; private final DataSize outputDataSize; private final DataSize failedOutputDataSize; private final long outputPositions; @@ -176,7 +174,7 @@ public StageStats( @JsonProperty("failedInputBlockedTime") Duration failedInputBlockedTime, @JsonProperty("bufferedDataSize") DataSize bufferedDataSize, - @JsonProperty("outputBufferUtilization") Optional outputBufferUtilization, + @JsonProperty("outputBufferUtilization") Optional outputBufferUtilization, @JsonProperty("outputDataSize") DataSize outputDataSize, @JsonProperty("failedOutputDataSize") DataSize failedOutputDataSize, @JsonProperty("outputPositions") long outputPositions, @@ -277,9 +275,7 @@ public StageStats( this.failedPhysicalWrittenDataSize = requireNonNull(failedPhysicalWrittenDataSize, "failedPhysicalWrittenDataSize is null"); this.gcInfo = requireNonNull(gcInfo, "gcInfo is null"); - - requireNonNull(operatorSummaries, "operatorSummaries is null"); - this.operatorSummaries = operatorSummaries.stream().map(OperatorStats::pruneDigests).collect(toImmutableList()); + this.operatorSummaries = ImmutableList.copyOf(operatorSummaries); } @JsonProperty @@ -547,7 +543,7 @@ public DataSize getBufferedDataSize() } @JsonProperty - public Optional getOutputBufferUtilization() + public Optional getOutputBufferUtilization() { return outputBufferUtilization; } @@ -661,69 +657,6 @@ public BasicStageStats toBasicStageStats(StageState stageState) runningPercentage); } - public StageStats pruneDigests() - { - return new StageStats( - schedulingComplete, - getSplitDistribution, - splitSourceMetrics, - totalTasks, - runningTasks, - completedTasks, - failedTasks, - totalDrivers, - queuedDrivers, - runningDrivers, - blockedDrivers, - completedDrivers, - cumulativeUserMemory, - failedCumulativeUserMemory, - userMemoryReservation, - revocableMemoryReservation, - totalMemoryReservation, - peakUserMemoryReservation, - peakRevocableMemoryReservation, - spilledDataSize, - totalScheduledTime, - failedScheduledTime, - totalCpuTime, - failedCpuTime, - totalBlockedTime, - fullyBlocked, - blockedReasons, - physicalInputDataSize, - failedPhysicalInputDataSize, - physicalInputPositions, - failedPhysicalInputPositions, - physicalInputReadTime, - failedPhysicalInputReadTime, - internalNetworkInputDataSize, - failedInternalNetworkInputDataSize, - internalNetworkInputPositions, - failedInternalNetworkInputPositions, - processedInputDataSize, - failedProcessedInputDataSize, - processedInputPositions, - failedProcessedInputPositions, - inputBlockedTime, - failedInputBlockedTime, - bufferedDataSize, - outputBufferUtilization, - outputDataSize, - failedOutputDataSize, - outputPositions, - failedOutputPositions, - pruneMetrics(outputBufferMetrics), - outputBlockedTime, - failedOutputBlockedTime, - physicalWrittenDataSize, - failedPhysicalWrittenDataSize, - gcInfo, - operatorSummaries.stream() - .map(OperatorStats::pruneDigests) - .collect(toImmutableList())); - } - public static StageStats createInitial() { DataSize zeroBytes = DataSize.of(0, BYTE); diff --git a/core/trino-main/src/main/java/io/trino/execution/StagesInfo.java b/core/trino-main/src/main/java/io/trino/execution/StagesInfo.java index 457bf0798e60..4567b7864afe 100644 --- a/core/trino-main/src/main/java/io/trino/execution/StagesInfo.java +++ b/core/trino-main/src/main/java/io/trino/execution/StagesInfo.java @@ -20,6 +20,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,13 +66,6 @@ public List getStages() return stages; } - public StagesInfo pruneDigests() - { - return new StagesInfo( - outputStageId, - stages.stream().map(StageInfo::pruneDigests).collect(toImmutableList())); - } - @JsonIgnore public StageInfo getOutputStage() { @@ -92,13 +86,13 @@ public List getSubStages(StageId stageId) } @JsonIgnore - public List getSubStagesDeepPreOrder(StageId stageId) + public List getSubStagesDeep(StageId stageId) { - return getSubStagesDeepPreOrder(stageId, false); + return getSubStagesDeep(stageId, false); } @JsonIgnore - public List getSubStagesDeepPreOrder(StageId root, boolean includeRoot) + public List getSubStagesDeep(StageId root, boolean includeRoot) { StageInfo stageInfo = stagesById.get(root); checkArgument(stageInfo != null, "stage %s not found", root); @@ -112,7 +106,7 @@ public List getSubStagesDeepPreOrder(StageId root, boolean includeRoo return subStagesIds.build().stream().map(stagesById::get).collect(toImmutableList()); } - private void collectSubStageIdsPreOrder(StageInfo stageInfo, ImmutableSet.Builder collector) + private void collectSubStageIdsPreOrder(StageInfo stageInfo, ImmutableSet.Builder collector) { stageInfo.getSubStages().stream().forEach(subStageId -> { collector.add(subStageId); @@ -122,33 +116,29 @@ private void collectSubStageIdsPreOrder(StageInfo stageInfo, ImmutableSet.Builde } @JsonIgnore - public List getSubStagesDeepPostOrder(StageId stageId) + public List getSubStagesDeepTopological(StageId root, boolean includeRoot) { - return getSubStagesDeepPostOrder(stageId, false); + ImmutableList.Builder builder = ImmutableList.builder(); + getSubStagesDeepTopologicalInner(root, builder, new HashSet<>(), includeRoot); + + return builder.build().reverse(); } - @JsonIgnore - public List getSubStagesDeepPostOrder(StageId root, boolean includeRoot) + private void getSubStagesDeepTopologicalInner(StageId stageId, ImmutableList.Builder builder, Set visitedFragments, boolean includeRoot) { - StageInfo stageInfo = stagesById.get(root); - checkArgument(stageInfo != null, "stage %s not found", root); - - ImmutableSet.Builder subStagesIds = ImmutableSet.builder(); - collectSubStageIdsPostOrder(stageInfo, subStagesIds); - if (includeRoot) { - subStagesIds.add(root); + if (visitedFragments.contains(stageId)) { + return; } - return subStagesIds.build().stream().map(stagesById::get).collect(toImmutableList()); - } + StageInfo stageInfo = stagesById.get(stageId); - private void collectSubStageIdsPostOrder(StageInfo stageInfo, ImmutableSet.Builder collector) - { - stageInfo.getSubStages().stream().forEach(subStageId -> { - StageInfo subStage = stagesById.get(subStageId); - collectSubStageIdsPostOrder(subStage, collector); - collector.add(subStageId); - }); + for (StageId childId : stageInfo.getSubStages().reversed()) { + getSubStagesDeepTopologicalInner(childId, builder, visitedFragments, true); + } + if (includeRoot) { + builder.add(stageInfo); + } + visitedFragments.add(stageId); } public static List getAllStages(Optional stages) diff --git a/core/trino-main/src/main/java/io/trino/execution/TaskId.java b/core/trino-main/src/main/java/io/trino/execution/TaskId.java index 938a647a7d2d..036129644e1b 100644 --- a/core/trino-main/src/main/java/io/trino/execution/TaskId.java +++ b/core/trino-main/src/main/java/io/trino/execution/TaskId.java @@ -18,94 +18,45 @@ import io.trino.spi.QueryId; import java.util.List; -import java.util.Objects; import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.SizeOf.estimatedSizeOf; import static io.airlift.slice.SizeOf.instanceSize; import static io.trino.spi.QueryId.parseDottedId; import static java.lang.Integer.parseInt; -import static java.lang.String.join; import static java.util.Objects.requireNonNull; -public class TaskId +public record TaskId(StageId stageId, int partitionId, int attemptId) { private static final int INSTANCE_SIZE = instanceSize(TaskId.class); @JsonCreator - public static TaskId valueOf(String taskId) + public static TaskId valueOf(String fullId) { - return new TaskId(taskId); + List parts = parseDottedId(fullId, 4, "taskId"); + return new TaskId(new StageId(new QueryId(parts.get(0)), parseInt(parts.get(1))), parseInt(parts.get(2)), parseInt(parts.get(3))); } - private final String fullId; - - public TaskId(StageId stageId, int partitionId, int attemptId) + public TaskId { requireNonNull(stageId, "stageId is null"); checkArgument(partitionId >= 0, "partitionId is negative: %s", partitionId); checkArgument(attemptId >= 0, "attemptId is negative: %s", attemptId); - - // There is a strange JDK bug related to the CompactStrings implementation in JDK20+ which causes some fullId values - // to get corrupted when this particular line is JIT-optimized. Changing implicit concatenation to a String.join call - // seems to mitigate this issue. See: https://github.com/trinodb/trino/issues/18272 for more details. - this.fullId = join(".", stageId.toString(), String.valueOf(partitionId), String.valueOf(attemptId)); - } - - private TaskId(String fullId) - { - this.fullId = requireNonNull(fullId, "fullId is null"); - } - - public QueryId getQueryId() - { - return new QueryId(parseDottedId(fullId, 4, "taskId").get(0)); - } - - public StageId getStageId() - { - List ids = parseDottedId(fullId, 4, "taskId"); - return StageId.valueOf(ids.subList(0, 2)); } - public int getPartitionId() + public QueryId queryId() { - return parseInt(parseDottedId(fullId, 4, "taskId").get(2)); - } - - public int getAttemptId() - { - return parseInt(parseDottedId(fullId, 4, "taskId").get(3)); + return stageId.queryId(); } @Override @JsonValue public String toString() { - return fullId; - } - - @Override - public int hashCode() - { - return Objects.hash(fullId); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - TaskId other = (TaskId) obj; - return Objects.equals(this.fullId, other.fullId); + return stageId.queryId().id() + '.' + stageId.id() + '.' + partitionId + '.' + attemptId; } public long getRetainedSizeInBytes() { - return INSTANCE_SIZE + estimatedSizeOf(fullId); + return INSTANCE_SIZE + stageId.getRetainedSizeInBytes(); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/TaskInfo.java b/core/trino-main/src/main/java/io/trino/execution/TaskInfo.java index 2ce9234395a8..a19879b97835 100644 --- a/core/trino-main/src/main/java/io/trino/execution/TaskInfo.java +++ b/core/trino-main/src/main/java/io/trino/execution/TaskInfo.java @@ -64,11 +64,6 @@ public TaskInfo pruneSpoolingOutputStats() return new TaskInfo(taskStatus, lastHeartbeat, outputBuffers.pruneSpoolingOutputStats(), noMoreSplits, stats, estimatedMemory, needsPlan); } - public TaskInfo pruneDigests() - { - return new TaskInfo(taskStatus, lastHeartbeat, outputBuffers.pruneDigests(), noMoreSplits, stats.pruneDigests(), estimatedMemory, needsPlan); - } - @Override public String toString() { diff --git a/core/trino-main/src/main/java/io/trino/execution/TaskStateMachine.java b/core/trino-main/src/main/java/io/trino/execution/TaskStateMachine.java index 4a255f1bebbd..29824b96bbae 100644 --- a/core/trino-main/src/main/java/io/trino/execution/TaskStateMachine.java +++ b/core/trino-main/src/main/java/io/trino/execution/TaskStateMachine.java @@ -20,6 +20,7 @@ import com.google.errorprone.annotations.concurrent.GuardedBy; import io.airlift.log.Logger; import io.trino.execution.StateMachine.StateChangeListener; +import jakarta.annotation.Nullable; import java.time.Instant; import java.util.ArrayList; @@ -28,6 +29,7 @@ import java.util.Map; import java.util.concurrent.Executor; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; @@ -61,13 +63,19 @@ public class TaskStateMachine private final Map sourceTaskFailures = new HashMap<>(); @GuardedBy("this") private final List sourceTaskFailureListeners = new ArrayList<>(); + private final AtomicReference executionEndTime = new AtomicReference<>(); public TaskStateMachine(TaskId taskId, Executor executor) { this.taskId = requireNonNull(taskId, "taskId is null"); this.executor = requireNonNull(executor, "executor is null"); taskState = new StateMachine<>("task " + taskId, executor, RUNNING, TERMINAL_TASK_STATES); - taskState.addStateChangeListener(newState -> log.debug("Task %s is %s", taskId, newState)); + taskState.addStateChangeListener(newState -> { + if (newState.isDone()) { + executionEndTime.compareAndSet(null, Instant.now()); + } + log.debug("Task %s is %s", taskId, newState); + }); } public Instant getCreatedTime() @@ -75,6 +83,16 @@ public Instant getCreatedTime() return createdTime; } + @Nullable + public Instant getEndTime() + { + if (getState().isDone()) { + executionEndTime.compareAndSet(null, Instant.now()); + return executionEndTime.get(); + } + return null; + } + public TaskId getTaskId() { return taskId; diff --git a/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferInfo.java b/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferInfo.java index 45f88834fd69..a3a3ab96ac2e 100644 --- a/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferInfo.java +++ b/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferInfo.java @@ -16,7 +16,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import io.trino.execution.DistributionSnapshot; import io.trino.plugin.base.metrics.TDigestHistogram; import io.trino.spi.metrics.Metrics; @@ -193,23 +192,6 @@ public OutputBufferInfo pruneSpoolingOutputStats() metrics); } - public OutputBufferInfo pruneDigests() - { - return new OutputBufferInfo( - type, - state, - canAddBuffers, - canAddPages, - totalBufferedBytes, - totalBufferedPages, - totalRowsSent, - totalPagesSent, - pipelinedBufferStates, - Optional.empty(), - spoolingOutputStats, - metrics.map(DistributionSnapshot::pruneMetrics)); - } - @Override public String toString() { diff --git a/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java b/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java index 2431cc5f01f5..7953eae1efc0 100644 --- a/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java +++ b/core/trino-main/src/main/java/io/trino/execution/buffer/OutputBufferMemoryManager.java @@ -39,7 +39,7 @@ * - the memory pool is exhausted */ @ThreadSafe -class OutputBufferMemoryManager +final class OutputBufferMemoryManager { private static final ListenableFuture NOT_BLOCKED = immediateVoidFuture(); @@ -90,7 +90,7 @@ public void updateMemoryUsage(long bytesAdded) ListenableFuture waitForMemory = null; SettableFuture notifyUnblocked = null; - long currentBufferedBytes; + final long currentBufferedBytes; synchronized (this) { // If closed is true, that means the task is completed. In that state, // the output buffers already ignore the newly added pages, and therefore @@ -99,9 +99,9 @@ public void updateMemoryUsage(long bytesAdded) return; } - currentBufferedBytes = bufferedBytes.updateAndGet(bytes -> { - long result = bytes + bytesAdded; - checkArgument(result >= 0, "bufferedBytes (%s) plus delta (%s) would be negative", bytes, bytesAdded); + currentBufferedBytes = bufferedBytes.accumulateAndGet(bytesAdded, (bufferedBytes, delta) -> { + long result = bufferedBytes + delta; + checkArgument(result >= 0, "bufferedBytes (%s) plus delta (%s) would be negative", bufferedBytes, delta); return result; }); ListenableFuture blockedOnMemory = memoryContext.setBytes(currentBufferedBytes); @@ -121,9 +121,12 @@ public void updateMemoryUsage(long bytesAdded) this.bufferBlockedFuture = null; } } - recordBufferUtilization(); + recordBufferUtilization(currentBufferedBytes); + } + // Reduce contention by reading first and only updating if the new value might become the maximum (uncommon) + if (currentBufferedBytes > peakMemoryUsage.get()) { + peakMemoryUsage.accumulateAndGet(currentBufferedBytes, Math::max); } - peakMemoryUsage.accumulateAndGet(currentBufferedBytes, Math::max); // Notify listeners outside of the critical section notifyListener(notifyUnblocked); if (waitForMemory != null) { @@ -131,13 +134,13 @@ public void updateMemoryUsage(long bytesAdded) } } - private synchronized void recordBufferUtilization() + private synchronized void recordBufferUtilization(long currentBufferedBytes) { long recordTime = ticker.read(); if (lastBufferUtilizationRecordTime != -1) { bufferUtilization.add(lastBufferUtilization, (double) recordTime - this.lastBufferUtilizationRecordTime); } - double utilization = getUtilization(); + double utilization = getUtilization(currentBufferedBytes); // skip recording of buffer utilization until data is put into buffer if (lastBufferUtilizationRecordTime != -1 || utilization != 0.0) { lastBufferUtilizationRecordTime = recordTime; @@ -187,13 +190,18 @@ public long getBufferedBytes() public double getUtilization() { - return bufferedBytes.get() / (double) maxBufferedBytes; + return getUtilization(bufferedBytes.get()); + } + + private double getUtilization(long currentBufferedBytes) + { + return currentBufferedBytes / (double) maxBufferedBytes; } public synchronized TDigest getUtilizationHistogram() { // always get most up to date histogram - recordBufferUtilization(); + recordBufferUtilization(bufferedBytes.get()); return TDigest.copyOf(bufferUtilization); } diff --git a/core/trino-main/src/main/java/io/trino/execution/executor/dedicated/SplitProcessor.java b/core/trino-main/src/main/java/io/trino/execution/executor/dedicated/SplitProcessor.java index bc0db84cdb59..474584e66e75 100644 --- a/core/trino-main/src/main/java/io/trino/execution/executor/dedicated/SplitProcessor.java +++ b/core/trino-main/src/main/java/io/trino/execution/executor/dedicated/SplitProcessor.java @@ -59,10 +59,10 @@ public void run(SchedulerContext context) { Span splitSpan = tracer.spanBuilder("split") .setParent(Context.current().with(split.getPipelineSpan())) - .setAttribute(TrinoAttributes.QUERY_ID, taskId.getQueryId().toString()) - .setAttribute(TrinoAttributes.STAGE_ID, taskId.getStageId().toString()) + .setAttribute(TrinoAttributes.QUERY_ID, taskId.queryId().toString()) + .setAttribute(TrinoAttributes.STAGE_ID, taskId.stageId().toString()) .setAttribute(TrinoAttributes.TASK_ID, taskId.toString()) - .setAttribute(TrinoAttributes.PIPELINE_ID, taskId.getStageId() + "-" + split.getPipelineId()) + .setAttribute(TrinoAttributes.PIPELINE_ID, taskId.stageId() + "-" + split.getPipelineId()) .setAttribute(TrinoAttributes.SPLIT_ID, taskId + "-" + splitId) .startSpan(); diff --git a/core/trino-main/src/main/java/io/trino/execution/executor/timesharing/TimeSharingTaskExecutor.java b/core/trino-main/src/main/java/io/trino/execution/executor/timesharing/TimeSharingTaskExecutor.java index 7d7d31a50b4a..17015ce7912f 100644 --- a/core/trino-main/src/main/java/io/trino/execution/executor/timesharing/TimeSharingTaskExecutor.java +++ b/core/trino-main/src/main/java/io/trino/execution/executor/timesharing/TimeSharingTaskExecutor.java @@ -358,10 +358,10 @@ public List> enqueueSplits(TaskHandle taskHandle, boolean Span splitSpan = tracer.spanBuilder(intermediate ? "split (intermediate)" : "split (leaf)") .setParent(Context.current().with(taskSplit.getPipelineSpan())) - .setAttribute(TrinoAttributes.QUERY_ID, taskId.getQueryId().toString()) - .setAttribute(TrinoAttributes.STAGE_ID, taskId.getStageId().toString()) + .setAttribute(TrinoAttributes.QUERY_ID, taskId.queryId().toString()) + .setAttribute(TrinoAttributes.STAGE_ID, taskId.stageId().toString()) .setAttribute(TrinoAttributes.TASK_ID, taskId.toString()) - .setAttribute(TrinoAttributes.PIPELINE_ID, taskId.getStageId() + "-" + taskSplit.getPipelineId()) + .setAttribute(TrinoAttributes.PIPELINE_ID, taskId.stageId() + "-" + taskSplit.getPipelineId()) .setAttribute(TrinoAttributes.SPLIT_ID, taskId + "-" + splitId) .startSpan(); diff --git a/core/trino-main/src/main/java/io/trino/execution/resourcegroups/InternalResourceGroup.java b/core/trino-main/src/main/java/io/trino/execution/resourcegroups/InternalResourceGroup.java index 9de719bbfeab..089024cb9aff 100644 --- a/core/trino-main/src/main/java/io/trino/execution/resourcegroups/InternalResourceGroup.java +++ b/core/trino-main/src/main/java/io/trino/execution/resourcegroups/InternalResourceGroup.java @@ -136,7 +136,7 @@ public class InternalResourceGroup @GuardedBy("root") private ResourceUsage cachedResourceUsage = new ResourceUsage(0, 0, 0); @GuardedBy("root") - private long lastStartMillis; + private long lastStartNanos; private final CounterStat timeBetweenStartsSec = new CounterStat(); private final CounterStat startedQueries = new CounterStat(); @@ -776,7 +776,7 @@ private void updateEligibility() } else { parent.get().eligibleSubGroups.remove(this); - lastStartMillis = 0; + lastStartNanos = 0; } parent.get().updateEligibility(); } @@ -970,11 +970,11 @@ private boolean internalStartNext() boolean started = subGroup.internalStartNext(); checkState(started, "Eligible sub group had no queries to run"); - long currentTime = System.currentTimeMillis(); - if (lastStartMillis != 0) { - timeBetweenStartsSec.update(Math.max(0, (currentTime - lastStartMillis) / 1000)); + long currentTime = System.nanoTime(); + if (lastStartNanos != 0) { + timeBetweenStartsSec.update(Math.max(0, (currentTime - lastStartNanos) / 1_000_000)); } - lastStartMillis = currentTime; + lastStartNanos = currentTime; descendantQueuedQueries--; // Don't call updateEligibility here, as we're in a recursive call, and don't want to repeatedly update our ancestors. diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/MultiSourcePartitionedScheduler.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/MultiSourcePartitionedScheduler.java index 1a20aa19e6fb..2d31bdc9b327 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/MultiSourcePartitionedScheduler.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/MultiSourcePartitionedScheduler.java @@ -96,7 +96,7 @@ public synchronized void start() * * there can be task in other stage blocked waiting for the dynamic filters, or * * connector split source for this stage might be blocked waiting the dynamic filters. */ - if (dynamicFilterService.isCollectingTaskNeeded(stageExecution.getStageId().getQueryId(), stageExecution.getFragment())) { + if (dynamicFilterService.isCollectingTaskNeeded(stageExecution.getStageId().queryId(), stageExecution.getFragment())) { stageExecution.beginScheduling(); /* * We can select node randomly because DynamicFilterSourceOperator is not dependent on splits diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedQueryScheduler.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedQueryScheduler.java index d228f37cd43a..3515ef5fb7cf 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedQueryScheduler.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedQueryScheduler.java @@ -1341,7 +1341,7 @@ else if (!result.getBlocked().isDone()) { // allow for schedule to resume scheduling (e.g. when some active stage completes // and dependent stages can be started) stagesScheduleResult.getRescheduleFuture().ifPresent(futures::add); - try (TimeStat.BlockTimer timer = schedulerStats.getSleepTime().time()) { + try (TimeStat.BlockTimer _ = schedulerStats.getSleepTime().time()) { tryGetFutureValue(whenAnyComplete(futures.build()), 1, SECONDS); } for (ListenableFuture blockedStage : blockedStages) { @@ -1361,17 +1361,12 @@ else if (!result.getBlocked().isDone()) { fail(t, Optional.empty()); } finally { - RuntimeException closeError = new RuntimeException(); for (StageScheduler scheduler : stageSchedulers.values()) { try { scheduler.close(); } catch (Throwable t) { fail(t, Optional.empty()); - // Self-suppression not permitted - if (closeError != t) { - closeError.addSuppressed(t); - } } } } @@ -1405,7 +1400,7 @@ public void fail(Throwable failureCause, Optional failedStageId) public void reportTaskFailure(TaskId taskId, Throwable failureCause) { - StageExecution stageExecution = stageExecutions.get(taskId.getStageId()); + StageExecution stageExecution = stageExecutions.get(taskId.stageId()); if (stageExecution == null) { return; } @@ -1416,7 +1411,7 @@ public void reportTaskFailure(TaskId taskId, Throwable failureCause) } stageExecution.failTask(taskId, failureCause); - stateMachine.transitionToFailed(failureCause, Optional.of(taskId.getStageId())); + stateMachine.transitionToFailed(failureCause, Optional.of(taskId.stageId())); stageExecutions.values().forEach(StageExecution::abort); } diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedStageExecution.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedStageExecution.java index d60983aa9937..16fdd233d0fc 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedStageExecution.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/PipelinedStageExecution.java @@ -277,7 +277,7 @@ public synchronized void fail(Throwable failureCause) @Override public synchronized void failTask(TaskId taskId, Throwable failureCause) { - RemoteTask task = requireNonNull(tasks.get(taskId.getPartitionId()), () -> "task not found: " + taskId); + RemoteTask task = requireNonNull(tasks.get(taskId.partitionId()), () -> "task not found: " + taskId); task.failLocallyImmediately(failureCause); fail(failureCause); } @@ -337,7 +337,7 @@ public synchronized Optional scheduleTask( taskLifecycleListener.taskCreated(stage.getFragment().getId(), task); // update output buffers - OutputBufferId outputBufferId = new OutputBufferId(task.getTaskId().getPartitionId()); + OutputBufferId outputBufferId = new OutputBufferId(task.getTaskId().partitionId()); updateSourceTasksOutputBuffers(outputBufferManager -> outputBufferManager.addOutputBuffer(outputBufferId)); return Optional.of(task); @@ -589,7 +589,7 @@ private static Split createExchangeSplit(RemoteTask sourceTask, RemoteTask desti { // Fetch the results from the buffer assigned to the task based on id URI exchangeLocation = sourceTask.getTaskStatus().getSelf(); - URI splitLocation = uriBuilderFrom(exchangeLocation).appendPath("results").appendPath(String.valueOf(destinationTask.getTaskId().getPartitionId())).build(); + URI splitLocation = uriBuilderFrom(exchangeLocation).appendPath("results").appendPath(String.valueOf(destinationTask.getTaskId().partitionId())).build(); return new Split(REMOTE_CATALOG_HANDLE, new RemoteSplit(new DirectExchangeInput(sourceTask.getTaskId(), splitLocation.toString()))); } diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/SourcePartitionedScheduler.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/SourcePartitionedScheduler.java index 7ebd561e881c..c03d9173d626 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/SourcePartitionedScheduler.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/SourcePartitionedScheduler.java @@ -218,7 +218,7 @@ public synchronized void start() // Avoid deadlocks by immediately scheduling a task for collecting dynamic filters because: // * there can be task in other stage blocked waiting for the dynamic filters, or // * connector split source for this stage might be blocked waiting the dynamic filters. - if (dynamicFilterService.isCollectingTaskNeeded(stageExecution.getStageId().getQueryId(), stageExecution.getFragment())) { + if (dynamicFilterService.isCollectingTaskNeeded(stageExecution.getStageId().queryId(), stageExecution.getFragment())) { stageExecution.beginScheduling(); createTaskOnRandomNode(); } @@ -316,7 +316,7 @@ else if (pendingSplits.isEmpty()) { Optional> tableExecuteSplitsInfo = splitSource.getTableExecuteSplitsInfo(); // Here we assume that we can get non-empty tableExecuteSplitsInfo only for queries which facilitate single split source. tableExecuteSplitsInfo.ifPresent(info -> { - TableExecuteContext tableExecuteContext = tableExecuteContextManager.getTableExecuteContextForQuery(stageExecution.getStageId().getQueryId()); + TableExecuteContext tableExecuteContext = tableExecuteContextManager.getTableExecuteContextForQuery(stageExecution.getStageId().queryId()); tableExecuteContext.setSplitsInfo(info); }); @@ -336,7 +336,7 @@ else if (pendingSplits.isEmpty()) { // Dynamic filters might not be collected due to build side source tasks being blocked on full buffer. // In such case probe split generation that is waiting for dynamic filters should be unblocked to prevent deadlock. log.debug("stage id: %s, node: %s; unblocking dynamic filters", stageExecution.getStageId(), partitionedNode); - dynamicFilterService.unblockStageDynamicFilters(stageExecution.getStageId().getQueryId(), stageExecution.getAttemptId(), stageExecution.getFragment()); + dynamicFilterService.unblockStageDynamicFilters(stageExecution.getStageId().queryId(), stageExecution.getAttemptId(), stageExecution.getFragment()); if (blockedOnPlacements) { // In a broadcast join, output buffers of the tasks in build source stage have to diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/StageManager.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/StageManager.java index d3e854773fc8..963df8eb5bf3 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/StageManager.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/StageManager.java @@ -175,7 +175,7 @@ public void abort() public void failTaskRemotely(TaskId taskId, Throwable failureCause) { - SqlStage sqlStage = requireNonNull(stages.get(taskId.getStageId()), () -> "stage not found: %s" + taskId.getStageId()); + SqlStage sqlStage = requireNonNull(stages.get(taskId.stageId()), () -> "stage not found: %s" + taskId.stageId()); sqlStage.failTaskRemotely(taskId, failureCause); } diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/BinPackingNodeAllocatorService.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/BinPackingNodeAllocatorService.java index 0381620ab38a..62d41ce17ed7 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/BinPackingNodeAllocatorService.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/BinPackingNodeAllocatorService.java @@ -114,7 +114,7 @@ public class BinPackingNodeAllocatorService private final DataSize eagerSpeculativeTasksNodeMemoryOvercommit; private final Ticker ticker; - private final ConcurrentNavigableMap> pendingAcquires = new ConcurrentSkipListMap<>(Ordering.natural().onResultOf(QueryId::getId)); + private final ConcurrentNavigableMap> pendingAcquires = new ConcurrentSkipListMap<>(Ordering.natural().onResultOf(QueryId::id)); private final Set fulfilledAcquires = newConcurrentHashSet(); private final Duration allowedNoMatchingNodePeriod; private final Duration exhaustedNodeWaitPeriod; diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/EventDrivenFaultTolerantQueryScheduler.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/EventDrivenFaultTolerantQueryScheduler.java index d6cbac15fe8e..a2abbcfb2e64 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/EventDrivenFaultTolerantQueryScheduler.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/EventDrivenFaultTolerantQueryScheduler.java @@ -560,7 +560,7 @@ public Duration getTotalCpuTime() public void failTaskRemotely(TaskId taskId, Throwable failureCause) { - SqlStage sqlStage = requireNonNull(stages.get(taskId.getStageId()), () -> "stage not found: %s" + taskId.getStageId()); + SqlStage sqlStage = requireNonNull(stages.get(taskId.stageId()), () -> "stage not found: %s" + taskId.stageId()); sqlStage.failTaskRemotely(taskId, failureCause); } @@ -582,13 +582,13 @@ private static class EventDebugInfos @Override public String onRemoteTaskEvent(RemoteTaskEvent event) { - return "task_" + event.getTaskStatus().getTaskId().getStageId().toString(); + return "task_" + event.getTaskStatus().getTaskId().stageId().toString(); } @Override public String onRemoteTaskExchangeUpdatedSinkAcquired(RemoteTaskExchangeUpdatedSinkAcquired event) { - return "task_" + event.getTaskId().getStageId().toString(); + return "task_" + event.getTaskId().stageId().toString(); } @Override @@ -1493,7 +1493,7 @@ private void createStageExecution( fragment.getOutputPartitioningScheme().getPartitionCount()); ExchangeContext exchangeContext = new ExchangeContextInstance( queryStateMachine.getQueryId(), - new ExchangeId("external-exchange-" + stage.getStageId().getId()), + new ExchangeId("external-exchange-" + stage.getStageId().id()), schedulerSpan); boolean preserveOrderWithinPartition = rootFragment && stage.getFragment().getPartitioning().equals(SINGLE_DISTRIBUTION); @@ -1816,7 +1816,7 @@ public Void onRemoteTaskCompleted(RemoteTaskCompletedEvent event) TaskStatus taskStatus = event.getTaskStatus(); TaskId taskId = taskStatus.getTaskId(); TaskState taskState = taskStatus.getState(); - StageExecution stageExecution = getStageExecution(taskId.getStageId()); + StageExecution stageExecution = getStageExecution(taskId.stageId()); if (stageExecution.getState().isDone()) { return null; @@ -1854,7 +1854,7 @@ else if (taskState == TaskState.FAILED) { public Void onRemoteTaskExchangeSinkUpdateRequired(RemoteTaskExchangeSinkUpdateRequiredEvent event) { TaskId taskId = event.getTaskStatus().getTaskId(); - StageExecution stageExecution = getStageExecution(taskId.getStageId()); + StageExecution stageExecution = getStageExecution(taskId.stageId()); stageExecution.initializeUpdateOfExchangeSinkInstanceHandle(taskId, eventQueue); return null; } @@ -1863,7 +1863,7 @@ public Void onRemoteTaskExchangeSinkUpdateRequired(RemoteTaskExchangeSinkUpdateR public Void onRemoteTaskExchangeUpdatedSinkAcquired(RemoteTaskExchangeUpdatedSinkAcquired event) { TaskId taskId = event.getTaskId(); - StageExecution stageExecution = getStageExecution(taskId.getStageId()); + StageExecution stageExecution = getStageExecution(taskId.stageId()); stageExecution.finalizeUpdateOfExchangeSinkInstanceHandle(taskId, event.getExchangeSinkInstanceHandle()); return null; } @@ -2443,12 +2443,12 @@ public void initializeUpdateOfExchangeSinkInstanceHandle(TaskId taskId, Blocking if (getState().isDone()) { return; } - StagePartition partition = getStagePartition(taskId.getPartitionId()); - CompletableFuture exchangeSinkInstanceHandleFuture = exchange.updateSinkInstanceHandle(partition.getExchangeSinkHandle(), taskId.getAttemptId()); + StagePartition partition = getStagePartition(taskId.partitionId()); + CompletableFuture exchangeSinkInstanceHandleFuture = exchange.updateSinkInstanceHandle(partition.getExchangeSinkHandle(), taskId.attemptId()); exchangeSinkInstanceHandleFuture.whenComplete((sinkInstanceHandle, throwable) -> { if (throwable != null) { - eventQueue.add(new StageFailureEvent(taskId.getStageId(), throwable)); + eventQueue.add(new StageFailureEvent(taskId.stageId(), throwable)); } else { eventQueue.add(new RemoteTaskExchangeUpdatedSinkAcquired(taskId, sinkInstanceHandle)); @@ -2461,7 +2461,7 @@ public void finalizeUpdateOfExchangeSinkInstanceHandle(TaskId taskId, ExchangeSi if (getState().isDone()) { return; } - StagePartition partition = getStagePartition(taskId.getPartitionId()); + StagePartition partition = getStagePartition(taskId.partitionId()); partition.updateExchangeSinkInstanceHandle(taskId, updatedExchangeSinkInstanceHandle); } @@ -2472,7 +2472,7 @@ public void finalizeUpdateOfExchangeSinkInstanceHandle(TaskId taskId, ExchangeSi */ public Optional> taskFinished(TaskId taskId, TaskStatus taskStatus) { - int partitionId = taskId.getPartitionId(); + int partitionId = taskId.partitionId(); StagePartition partition = getStagePartition(partitionId); Optional outputStats = partition.taskFinished(taskId); @@ -2483,7 +2483,7 @@ public Optional> taskFinished(TaskId taskId, Task return Optional.of(taskFailed(taskId, Failures.toFailure(new TrinoException(GENERIC_INTERNAL_ERROR, "Treating FINISHED task as FAILED because we received empty spooling output stats")), taskStatus)); } - exchange.sinkFinished(partition.getExchangeSinkHandle(), taskId.getAttemptId()); + exchange.sinkFinished(partition.getExchangeSinkHandle(), taskId.attemptId()); if (!partition.isRunning()) { runningPartitions.remove(partitionId); @@ -2502,7 +2502,7 @@ public Optional> taskFinished(TaskId taskId, Task true, Optional.empty()); - sinkOutputSelectorBuilder.include(exchange.getId(), taskId.getPartitionId(), taskId.getAttemptId()); + sinkOutputSelectorBuilder.include(exchange.getId(), taskId.partitionId(), taskId.attemptId()); if (noMorePartitions && remainingPartitions.isEmpty() && !stage.getState().isDone()) { finish(); @@ -2568,7 +2568,7 @@ private void updateOutputSize(SpoolingOutputStats.Snapshot taskOutputStats) public List taskFailed(TaskId taskId, ExecutionFailureInfo failureInfo, TaskStatus taskStatus) { - int partitionId = taskId.getPartitionId(); + int partitionId = taskId.partitionId(); StagePartition partition = getStagePartition(partitionId); partition.taskFailed(taskId); diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/TaskDescriptorStorage.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/TaskDescriptorStorage.java index 12e2bfc2f302..2f37bd4ead55 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/TaskDescriptorStorage.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/faulttolerant/TaskDescriptorStorage.java @@ -217,7 +217,7 @@ public synchronized void initialize(QueryId queryId) */ public synchronized void put(StageId stageId, TaskDescriptor descriptor) { - TaskDescriptors storage = storages.get(stageId.getQueryId()); + TaskDescriptors storage = storages.get(stageId.queryId()); if (storage == null) { // query has been terminated return; @@ -256,7 +256,7 @@ private void runAndUpdateMemory(TaskDescriptors storage, Runnable operation, boo */ public synchronized Optional get(StageId stageId, int partitionId) { - TaskDescriptors storage = storages.get(stageId.getQueryId()); + TaskDescriptors storage = storages.get(stageId.queryId()); if (storage == null) { // query has been terminated return Optional.empty(); @@ -272,7 +272,7 @@ public synchronized Optional get(StageId stageId, int partitionI */ public synchronized void remove(StageId stageId, int partitionId) { - TaskDescriptors storage = storages.get(stageId.getQueryId()); + TaskDescriptors storage = storages.get(stageId.queryId()); if (storage == null) { // query has been terminated return; diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/policy/PhasedExecutionSchedule.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/policy/PhasedExecutionSchedule.java index beb8be903f24..66844226ec1e 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/policy/PhasedExecutionSchedule.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/policy/PhasedExecutionSchedule.java @@ -285,7 +285,7 @@ private Set extractDependenciesAndReturnNonLazyFragments(Collect } QueryId queryId = stages.stream() - .map(stage -> stage.getStageId().getQueryId()) + .map(stage -> stage.getStageId().queryId()) .distinct() .collect(onlyElement()); List fragments = stages.stream() diff --git a/core/trino-main/src/main/java/io/trino/memory/ClusterMemoryManager.java b/core/trino-main/src/main/java/io/trino/memory/ClusterMemoryManager.java index 1c99dbdb4e7d..d118d0e56ebf 100644 --- a/core/trino-main/src/main/java/io/trino/memory/ClusterMemoryManager.java +++ b/core/trino-main/src/main/java/io/trino/memory/ClusterMemoryManager.java @@ -271,7 +271,7 @@ private synchronized void callOomKiller(Iterable runningQueries) log.debug("Low memory killer chose %s", tasks); ImmutableSet.Builder killedTasksBuilder = ImmutableSet.builder(); for (TaskId task : tasks) { - Optional runningQuery = findRunningQuery(runningQueries, task.getQueryId()); + Optional runningQuery = findRunningQuery(runningQueries, task.queryId()); if (runningQuery.isPresent()) { runningQuery.get().failTask(task, new TrinoException(CLUSTER_OUT_OF_MEMORY, "Task killed because the cluster is out of memory.")); tasksKilledDueToOutOfMemory.incrementAndGet(); diff --git a/core/trino-main/src/main/java/io/trino/memory/LeastWastedEffortTaskLowMemoryKiller.java b/core/trino-main/src/main/java/io/trino/memory/LeastWastedEffortTaskLowMemoryKiller.java index 8456a558f7b9..02ea6a58f089 100644 --- a/core/trino-main/src/main/java/io/trino/memory/LeastWastedEffortTaskLowMemoryKiller.java +++ b/core/trino-main/src/main/java/io/trino/memory/LeastWastedEffortTaskLowMemoryKiller.java @@ -85,7 +85,7 @@ private static Optional findBiggestTask(Set queriesWithTaskRetr { Stream> stream = memoryPool.getTaskMemoryReservations().entrySet().stream() .map(entry -> new SimpleEntry<>(TaskId.valueOf(entry.getKey()), entry.getValue())) - .filter(entry -> queriesWithTaskRetryPolicy.contains(entry.getKey().getQueryId())); + .filter(entry -> queriesWithTaskRetryPolicy.contains(entry.getKey().queryId())); if (onlySpeculative) { stream = stream.filter(entry -> { diff --git a/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java b/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java index a54837205d60..f1661ef6da0e 100644 --- a/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java +++ b/core/trino-main/src/main/java/io/trino/memory/LocalMemoryManager.java @@ -17,11 +17,14 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.inject.Inject; +import com.sun.management.UnixOperatingSystemMXBean; import io.airlift.units.DataSize; +import java.lang.management.ManagementFactory; import java.util.concurrent.TimeUnit; import static com.google.common.base.Verify.verify; +import static java.lang.Math.clamp; import static java.lang.String.format; public final class LocalMemoryManager @@ -31,6 +34,10 @@ public final class LocalMemoryManager private static final Supplier AVAILABLE_PROCESSORS = Suppliers .memoizeWithExpiration(Runtime.getRuntime()::availableProcessors, 30, TimeUnit.SECONDS); + // Clamp value because according to the documentation: if the recent CPU usage is not available, the method returns a negative value. + private static final Supplier SYSTEM_CPU_LOAD = Suppliers + .memoizeWithExpiration(() -> clamp(((UnixOperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean()).getCpuLoad(), 0.0, 1.0), 5, TimeUnit.SECONDS); + @Inject public LocalMemoryManager(NodeMemoryConfig config) { @@ -63,7 +70,7 @@ private void validateHeapHeadroom(NodeMemoryConfig config, long availableMemory) public MemoryInfo getInfo() { - return new MemoryInfo(AVAILABLE_PROCESSORS.get(), memoryPool.getInfo()); + return new MemoryInfo(AVAILABLE_PROCESSORS.get(), SYSTEM_CPU_LOAD.get(), memoryPool.getInfo()); } public MemoryPool getMemoryPool() diff --git a/core/trino-main/src/main/java/io/trino/memory/MemoryInfo.java b/core/trino-main/src/main/java/io/trino/memory/MemoryInfo.java index 86d58ce69ef1..66944882d84b 100644 --- a/core/trino-main/src/main/java/io/trino/memory/MemoryInfo.java +++ b/core/trino-main/src/main/java/io/trino/memory/MemoryInfo.java @@ -23,14 +23,17 @@ public class MemoryInfo { private final int availableProcessors; + private final double systemCpuLoad; private final MemoryPoolInfo pool; @JsonCreator public MemoryInfo( @JsonProperty("availableProcessors") int availableProcessors, + @JsonProperty("systemCpuLoad") double systemCpuLoad, @JsonProperty("pool") MemoryPoolInfo pool) { this.availableProcessors = availableProcessors; + this.systemCpuLoad = systemCpuLoad; this.pool = requireNonNull(pool, "pool is null"); } @@ -40,6 +43,12 @@ public int getAvailableProcessors() return availableProcessors; } + @JsonProperty + public double getSystemCpuLoad() + { + return systemCpuLoad; + } + @JsonProperty public MemoryPoolInfo getPool() { @@ -51,6 +60,7 @@ public String toString() { return toStringHelper(this) .add("availableProcessors", availableProcessors) + .add("systemCpuLoad", systemCpuLoad) .add("pool", pool) .toString(); } diff --git a/core/trino-main/src/main/java/io/trino/memory/MemoryPool.java b/core/trino-main/src/main/java/io/trino/memory/MemoryPool.java index fc5395330240..b6cf4c315182 100644 --- a/core/trino-main/src/main/java/io/trino/memory/MemoryPool.java +++ b/core/trino-main/src/main/java/io/trino/memory/MemoryPool.java @@ -130,7 +130,7 @@ public ListenableFuture reserve(TaskId taskId, String allocationTag, long ListenableFuture result; synchronized (this) { if (bytes != 0) { - QueryId queryId = taskId.getQueryId(); + QueryId queryId = taskId.queryId(); queryMemoryReservations.merge(queryId, bytes, Long::sum); updateTaggedMemoryAllocations(queryId, allocationTag, bytes); taskMemoryReservations.merge(taskId, bytes, Long::sum); @@ -164,7 +164,7 @@ public ListenableFuture reserveRevocable(TaskId taskId, long bytes) ListenableFuture result; synchronized (this) { if (bytes != 0) { - queryRevocableMemoryReservations.merge(taskId.getQueryId(), bytes, Long::sum); + queryRevocableMemoryReservations.merge(taskId.queryId(), bytes, Long::sum); taskRevocableMemoryReservations.merge(taskId, bytes, Long::sum); } reservedRevocableBytes += bytes; @@ -196,7 +196,7 @@ public boolean tryReserve(TaskId taskId, String allocationTag, long bytes) } reservedBytes += bytes; if (bytes != 0) { - QueryId queryId = taskId.getQueryId(); + QueryId queryId = taskId.queryId(); queryMemoryReservations.merge(queryId, bytes, Long::sum); updateTaggedMemoryAllocations(queryId, allocationTag, bytes); taskMemoryReservations.merge(taskId, bytes, Long::sum); @@ -230,7 +230,7 @@ public synchronized void free(TaskId taskId, String allocationTag, long bytes) return; } - QueryId queryId = taskId.getQueryId(); + QueryId queryId = taskId.queryId(); Long queryReservation = queryMemoryReservations.get(queryId); requireNonNull(queryReservation, "queryReservation is null"); checkArgument(queryReservation >= bytes, "tried to free more memory than is reserved by query"); @@ -273,7 +273,7 @@ public synchronized void freeRevocable(TaskId taskId, long bytes) return; } - QueryId queryId = taskId.getQueryId(); + QueryId queryId = taskId.queryId(); Long queryReservation = queryRevocableMemoryReservations.get(queryId); requireNonNull(queryReservation, "queryReservation is null"); checkArgument(queryReservation >= bytes, "tried to free more revocable memory than is reserved by query"); diff --git a/core/trino-main/src/main/java/io/trino/memory/TotalReservationOnBlockedNodesTaskLowMemoryKiller.java b/core/trino-main/src/main/java/io/trino/memory/TotalReservationOnBlockedNodesTaskLowMemoryKiller.java index b4550278ea8b..caba95db8786 100644 --- a/core/trino-main/src/main/java/io/trino/memory/TotalReservationOnBlockedNodesTaskLowMemoryKiller.java +++ b/core/trino-main/src/main/java/io/trino/memory/TotalReservationOnBlockedNodesTaskLowMemoryKiller.java @@ -74,12 +74,12 @@ private static Optional findBiggestTask(Map r Stream> stream = memoryPool.getTaskMemoryReservations().entrySet().stream() // consider only tasks from queries with task retries enabled .map(entry -> new SimpleEntry<>(TaskId.valueOf(entry.getKey()), entry.getValue())) - .filter(entry -> runningQueries.containsKey(entry.getKey().getQueryId())) - .filter(entry -> runningQueries.get(entry.getKey().getQueryId()).getRetryPolicy() == TASK); + .filter(entry -> runningQueries.containsKey(entry.getKey().queryId())) + .filter(entry -> runningQueries.get(entry.getKey().queryId()).getRetryPolicy() == TASK); if (onlySpeculative) { stream = stream.filter(entry -> { - TaskInfo taskInfo = runningQueries.get(entry.getKey().getQueryId()).getTaskInfos().get(entry.getKey()); + TaskInfo taskInfo = runningQueries.get(entry.getKey().queryId()).getTaskInfos().get(entry.getKey()); if (taskInfo == null) { return false; } diff --git a/core/trino-main/src/main/java/io/trino/metadata/Catalog.java b/core/trino-main/src/main/java/io/trino/metadata/Catalog.java index 5d82cb9184dc..c08f0429f1de 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/Catalog.java +++ b/core/trino-main/src/main/java/io/trino/metadata/Catalog.java @@ -28,6 +28,7 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static io.trino.connector.CatalogHandle.createRootCatalogHandle; +import static io.trino.metadata.CatalogStatus.FAILING; import static io.trino.spi.StandardErrorCode.CATALOG_UNAVAILABLE; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -40,7 +41,7 @@ public class Catalog private final ConnectorServices catalogConnector; private final ConnectorServices informationSchemaConnector; private final ConnectorServices systemConnector; - private final boolean loaded; + private final CatalogStatus catalogStatus; public Catalog( CatalogName catalogName, @@ -49,7 +50,7 @@ public Catalog( ConnectorServices catalogConnector, ConnectorServices informationSchemaConnector, ConnectorServices systemConnector, - boolean loaded) + CatalogStatus catalogStatus) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); this.catalogHandle = requireNonNull(catalogHandle, "catalogHandle is null"); @@ -58,15 +59,15 @@ public Catalog( this.catalogConnector = requireNonNull(catalogConnector, "catalogConnector is null"); this.informationSchemaConnector = requireNonNull(informationSchemaConnector, "informationSchemaConnector is null"); this.systemConnector = requireNonNull(systemConnector, "systemConnector is null"); - this.loaded = loaded; + this.catalogStatus = requireNonNull(catalogStatus, "catalogStatus is null"); } public static Catalog failedCatalog(CatalogName catalogName, CatalogVersion catalogVersion, ConnectorName connectorName) { - return new Catalog(catalogName, createRootCatalogHandle(catalogName, catalogVersion), connectorName, false); + return new Catalog(catalogName, createRootCatalogHandle(catalogName, catalogVersion), connectorName, FAILING); } - private Catalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName, boolean loaded) + private Catalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorName connectorName, CatalogStatus catalogStatus) { this.catalogName = catalogName; this.catalogHandle = catalogHandle; @@ -74,7 +75,7 @@ private Catalog(CatalogName catalogName, CatalogHandle catalogHandle, ConnectorN this.catalogConnector = null; this.informationSchemaConnector = null; this.systemConnector = null; - this.loaded = loaded; + this.catalogStatus = catalogStatus; } public CatalogName getCatalogName() @@ -92,9 +93,9 @@ public ConnectorName getConnectorName() return connectorName; } - public boolean isLoaded() + public CatalogStatus getCatalogStatus() { - return loaded; + return catalogStatus; } public boolean isFailed() diff --git a/core/trino-main/src/main/java/io/trino/metadata/CatalogInfo.java b/core/trino-main/src/main/java/io/trino/metadata/CatalogInfo.java index 904d95a18a12..df38ff8d43a5 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/CatalogInfo.java +++ b/core/trino-main/src/main/java/io/trino/metadata/CatalogInfo.java @@ -18,12 +18,17 @@ import static java.util.Objects.requireNonNull; -public record CatalogInfo(String catalogName, CatalogHandle catalogHandle, ConnectorName connectorName, boolean loaded) +public record CatalogInfo( + String catalogName, + CatalogHandle catalogHandle, + ConnectorName connectorName, + CatalogStatus catalogStatus) { public CatalogInfo { requireNonNull(catalogName, "catalogName is null"); requireNonNull(catalogHandle, "catalogHandle is null"); requireNonNull(connectorName, "connectorName is null"); + requireNonNull(catalogStatus, "catalogStatus is null"); } } diff --git a/core/trino-main/src/main/java/io/trino/metadata/CatalogStatus.java b/core/trino-main/src/main/java/io/trino/metadata/CatalogStatus.java new file mode 100644 index 000000000000..5cdcd654f182 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/metadata/CatalogStatus.java @@ -0,0 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.metadata; + +public enum CatalogStatus +{ + OPERATIONAL, + FAILING, +} diff --git a/core/trino-main/src/main/java/io/trino/metadata/DisabledSystemSecurityMetadata.java b/core/trino-main/src/main/java/io/trino/metadata/DisabledSystemSecurityMetadata.java index 653e0936d8d7..dec9c7c70566 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/DisabledSystemSecurityMetadata.java +++ b/core/trino-main/src/main/java/io/trino/metadata/DisabledSystemSecurityMetadata.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableSet; import io.trino.Session; import io.trino.spi.TrinoException; +import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.CatalogSchemaName; import io.trino.spi.connector.CatalogSchemaTableName; import io.trino.spi.connector.EntityKindAndName; @@ -198,6 +199,12 @@ public Optional getFunctionRunAsIdentity(Session session, CatalogSchem return Optional.empty(); } + @Override + public void catalogCreated(Session session, CatalogName catalog) {} + + @Override + public void catalogDropped(Session session, CatalogName catalog) {} + @Override public void functionCreated(Session session, CatalogSchemaFunctionName function) {} diff --git a/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java b/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java index 2b6fadb85c05..19786db52270 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java +++ b/core/trino-main/src/main/java/io/trino/metadata/FunctionManager.java @@ -184,9 +184,7 @@ private FunctionProvider getFunctionProvider(ResolvedFunction resolvedFunction) return globalFunctionCatalog; } - FunctionProvider functionProvider = functionProviders.getService(resolvedFunction.catalogHandle()); - checkArgument(functionProvider != null, "No function provider for catalog: '%s' (function '%s')", resolvedFunction.catalogHandle(), resolvedFunction.signature().getName()); - return functionProvider; + return functionProviders.getService(resolvedFunction.catalogHandle()); } private static void verifyMethodHandleSignature(BoundSignature boundSignature, ScalarFunctionImplementation scalarFunctionImplementation, InvocationConvention convention) diff --git a/core/trino-main/src/main/java/io/trino/metadata/Metadata.java b/core/trino-main/src/main/java/io/trino/metadata/Metadata.java index e931cc74f009..d5732b0f7b9a 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/Metadata.java +++ b/core/trino-main/src/main/java/io/trino/metadata/Metadata.java @@ -20,6 +20,7 @@ import io.trino.connector.CatalogHandle; import io.trino.spi.RefreshType; import io.trino.spi.TrinoException; +import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.AggregateFunction; import io.trino.spi.connector.AggregationApplicationResult; import io.trino.spi.connector.BeginTableExecuteResult; @@ -29,6 +30,7 @@ import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ColumnPosition; import io.trino.spi.connector.ConnectorCapabilities; +import io.trino.spi.connector.ConnectorName; import io.trino.spi.connector.ConnectorOutputMetadata; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.Constraint; @@ -65,6 +67,7 @@ import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.LanguageFunction; import io.trino.spi.function.OperatorType; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.FunctionAuthorization; import io.trino.spi.security.GrantInfo; @@ -121,7 +124,7 @@ Optional getTableHandleForExecute( void finishTableExecute(Session session, TableExecuteHandle handle, Collection fragments, List tableExecuteState); - void executeTableExecute(Session session, TableExecuteHandle handle); + Map executeTableExecute(Session session, TableExecuteHandle handle); TableProperties getTableProperties(Session session, TableHandle handle); @@ -145,6 +148,11 @@ Optional getTableHandleForExecute( Optional getInfo(Session session, TableHandle handle); + /** + * Return connector-specific, metadata operations metrics for the given session. + */ + Metrics getMetrics(Session session, String catalogName); + CatalogSchemaTableName getTableName(Session session, TableHandle tableHandle); /** @@ -208,6 +216,16 @@ Optional getTableHandleForExecute( */ List listRelationComments(Session session, String catalogName, Optional schemaName, UnaryOperator> relationFilter); + /** + * Creates a catalog. + */ + void createCatalog(Session session, CatalogName catalog, ConnectorName connectorName, Map properties, boolean notExists); + + /** + * Drops the specified catalog. + */ + void dropCatalog(Session session, CatalogName catalog, boolean cascade); + /** * Creates a schema. * @@ -341,7 +359,7 @@ Optional getTableHandleForExecute( /** * Describes statistics that must be collected during a write. */ - TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata); + TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata, boolean tableReplace); /** * Describe statistics that must be collected during a statistics collection @@ -467,10 +485,12 @@ Optional finishRefreshMaterializedView( Optional getCatalogHandle(Session session, String catalogName); /** - * Lists all defined catalogs (both loaded properly and failed ones). + * Gets all the catalogs */ List listCatalogs(Session session); + List listActiveCatalogs(Session session); + /** * Get the names that match the specified table prefix (never null). */ diff --git a/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java b/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java index fa0a9067c148..27fdb632f65e 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java +++ b/core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java @@ -35,6 +35,7 @@ import io.trino.spi.QueryId; import io.trino.spi.RefreshType; import io.trino.spi.TrinoException; +import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.AggregateFunction; import io.trino.spi.connector.AggregationApplicationResult; import io.trino.spi.connector.Assignment; @@ -50,6 +51,7 @@ import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorMergeTableHandle; import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorName; import io.trino.spi.connector.ConnectorOutputMetadata; import io.trino.spi.connector.ConnectorOutputTableHandle; import io.trino.spi.connector.ConnectorPartitioningHandle; @@ -102,6 +104,7 @@ import io.trino.spi.function.OperatorType; import io.trino.spi.function.SchemaFunctionName; import io.trino.spi.function.Signature; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.FunctionAuthorization; import io.trino.spi.security.GrantInfo; @@ -153,6 +156,7 @@ import static io.trino.SystemSessionProperties.getRetryPolicy; import static io.trino.metadata.CatalogMetadata.SecurityManagement.CONNECTOR; import static io.trino.metadata.CatalogMetadata.SecurityManagement.SYSTEM; +import static io.trino.metadata.CatalogStatus.OPERATIONAL; import static io.trino.metadata.GlobalFunctionCatalog.BUILTIN_SCHEMA; import static io.trino.metadata.GlobalFunctionCatalog.isBuiltinFunctionName; import static io.trino.metadata.LanguageFunctionManager.isTrinoSqlLanguageFunction; @@ -196,6 +200,7 @@ public final class MetadataManager private final TableFunctionRegistry tableFunctionRegistry; private final TypeManager typeManager; private final TypeCoercion typeCoercion; + private final CatalogManager catalogManager; private final ConcurrentMap catalogsByQueryId = new ConcurrentHashMap<>(); @@ -207,13 +212,15 @@ public MetadataManager( GlobalFunctionCatalog globalFunctionCatalog, LanguageFunctionManager languageFunctionManager, TableFunctionRegistry tableFunctionRegistry, - TypeManager typeManager) + TypeManager typeManager, + CatalogManager catalogManager) { this.accessControl = requireNonNull(accessControl, "accessControl is null"); this.typeManager = requireNonNull(typeManager, "typeManager is null"); functions = requireNonNull(globalFunctionCatalog, "globalFunctionCatalog is null"); functionResolver = new BuiltinFunctionResolver(this, typeManager, globalFunctionCatalog); this.typeCoercion = new TypeCoercion(typeManager::getType); + this.catalogManager = requireNonNull(catalogManager, "catalogManager is null"); this.systemSecurityMetadata = requireNonNull(systemSecurityMetadata, "systemSecurityMetadata is null"); this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); @@ -362,11 +369,11 @@ public void finishTableExecute(Session session, TableExecuteHandle tableExecuteH } @Override - public void executeTableExecute(Session session, TableExecuteHandle tableExecuteHandle) + public Map executeTableExecute(Session session, TableExecuteHandle tableExecuteHandle) { CatalogHandle catalogHandle = tableExecuteHandle.catalogHandle(); ConnectorMetadata metadata = getMetadata(session, catalogHandle); - metadata.executeTableExecute(session.toConnectorSession(catalogHandle), tableExecuteHandle.connectorHandle()); + return metadata.executeTableExecute(session.toConnectorSession(catalogHandle), tableExecuteHandle.connectorHandle()); } @Override @@ -451,6 +458,14 @@ public Optional getInfo(Session session, TableHandle handle) return metadata.getInfo(connectorSession, handle.connectorHandle()); } + @Override + public Metrics getMetrics(Session session, String catalogName) + { + return transactionManager.getRequiredCatalogMetadata(session.getRequiredTransactionId(), catalogName) + .getMetadata(session) + .getMetrics(session.toConnectorSession()); + } + @Override public CatalogSchemaTableName getTableName(Session session, TableHandle tableHandle) { @@ -793,6 +808,31 @@ public List listRelationComments(Session session, Strin return tableComments.build(); } + @Override + public void createCatalog(Session session, CatalogName catalog, ConnectorName connectorName, Map properties, boolean notExists) + { + catalogManager.createCatalog(catalog, connectorName, properties, notExists); + CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalog.toString()); + if (catalogMetadata.getSecurityManagement() == SYSTEM) { + systemSecurityMetadata.catalogCreated(session, catalog); + } + } + + @Override + public void dropCatalog(Session session, CatalogName catalog, boolean cascade) + { + Optional catalogMetadata = Optional.empty(); + // there is a potential race condition here, TODO: https://github.com/trinodb/trino/issues/26927 + Optional optionalCatalog = catalogManager.getCatalog(catalog); + if (optionalCatalog.isPresent() && optionalCatalog.get().getCatalogStatus() == OPERATIONAL) { + catalogMetadata = Optional.of(getCatalogMetadataForWrite(session, catalog.toString())); + } + catalogManager.dropCatalog(catalog, cascade); + if (catalogMetadata.isPresent() && catalogMetadata.get().getSecurityManagement() == SYSTEM) { + systemSecurityMetadata.catalogDropped(session, catalog); + } + } + @Override public void createSchema(Session session, CatalogSchemaName schema, Map properties, TrinoPrincipal principal) { @@ -1036,11 +1076,11 @@ public Optional getInsertLayout(Session session, TableHandle table) } @Override - public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata) + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata, boolean tableReplace) { CatalogMetadata catalogMetadata = getCatalogMetadataForWrite(session, catalogHandle); ConnectorMetadata metadata = catalogMetadata.getMetadata(session); - return metadata.getStatisticsCollectionMetadataForWrite(session.toConnectorSession(catalogHandle), tableMetadata); + return metadata.getStatisticsCollectionMetadataForWrite(session.toConnectorSession(catalogHandle), tableMetadata, tableReplace); } @Override @@ -1358,6 +1398,12 @@ public List listCatalogs(Session session) return transactionManager.getCatalogs(session.getRequiredTransactionId()); } + @Override + public List listActiveCatalogs(Session session) + { + return transactionManager.getActiveCatalogs(session.getRequiredTransactionId()); + } + @Override public List listViews(Session session, QualifiedTablePrefix prefix) { diff --git a/core/trino-main/src/main/java/io/trino/metadata/SystemSecurityMetadata.java b/core/trino-main/src/main/java/io/trino/metadata/SystemSecurityMetadata.java index 20a0f0c1cd27..0c64751e5a99 100644 --- a/core/trino-main/src/main/java/io/trino/metadata/SystemSecurityMetadata.java +++ b/core/trino-main/src/main/java/io/trino/metadata/SystemSecurityMetadata.java @@ -14,6 +14,7 @@ package io.trino.metadata; import io.trino.Session; +import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.CatalogSchemaName; import io.trino.spi.connector.CatalogSchemaTableName; import io.trino.spi.connector.EntityKindAndName; @@ -187,6 +188,16 @@ default void validateEntityKindAndPrivileges(Session session, String entityKind, */ Optional getFunctionRunAsIdentity(Session session, CatalogSchemaFunctionName functionName); + /** + * A catalog was created + */ + void catalogCreated(Session session, CatalogName catalog); + + /** + * A catalog was dropped + */ + void catalogDropped(Session session, CatalogName catalog); + /** * A function is created */ diff --git a/core/trino-main/src/main/java/io/trino/node/AirliftNodeInventoryModule.java b/core/trino-main/src/main/java/io/trino/node/AirliftNodeInventoryModule.java index 3e87485dccaa..087fa5d5fe6b 100644 --- a/core/trino-main/src/main/java/io/trino/node/AirliftNodeInventoryModule.java +++ b/core/trino-main/src/main/java/io/trino/node/AirliftNodeInventoryModule.java @@ -15,7 +15,6 @@ import com.google.inject.Binder; import com.google.inject.Scopes; -import com.google.inject.multibindings.Multibinder; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.discovery.client.DiscoveryModule; import io.airlift.discovery.client.ForDiscoveryClient; @@ -23,27 +22,12 @@ import io.airlift.discovery.server.EmbeddedDiscoveryModule; import io.airlift.discovery.server.ServiceResource; import io.airlift.discovery.store.StoreResource; -import io.airlift.http.client.HttpClientConfig; -import io.airlift.http.client.HttpRequestFilter; -import io.airlift.http.client.Request; import io.trino.failuredetector.FailureDetectorModule; -import io.trino.server.InternalAuthenticationManager; -import io.trino.server.InternalCommunicationConfig; import io.trino.server.NodeResource; import io.trino.server.ServerConfig; -import java.io.UncheckedIOException; -import java.net.InetAddress; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.UnknownHostException; - -import static com.google.inject.multibindings.Multibinder.newSetBinder; -import static io.airlift.configuration.ConfigBinder.configBinder; import static io.airlift.discovery.client.DiscoveryBinder.discoveryBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; -import static io.airlift.node.AddressToHostname.encodeAddressAsHostname; -import static io.trino.server.InternalCommunicationHttpClientModule.configureClient; import static io.trino.server.security.ResourceSecurityBinder.resourceSecurityBinder; public class AirliftNodeInventoryModule @@ -92,43 +76,6 @@ protected void setup(Binder binder) .addProperty("coordinator", String.valueOf(coordinator)); // internal communication setup for discovery http client - InternalCommunicationConfig internalCommunicationConfig = buildConfigObject(InternalCommunicationConfig.class); - Multibinder discoveryFilterBinder = newSetBinder(binder, HttpRequestFilter.class, ForDiscoveryClient.class); - if (internalCommunicationConfig.isHttpsRequired() && internalCommunicationConfig.getKeyStorePath() == null && internalCommunicationConfig.getTrustStorePath() == null) { - discoveryFilterBinder.addBinding().to(DiscoveryEncodeAddressAsHostname.class); - } - configBinder(binder).bindConfigDefaults(HttpClientConfig.class, ForDiscoveryClient.class, config -> configureClient(config, internalCommunicationConfig)); - discoveryFilterBinder.addBinding().to(InternalAuthenticationManager.class); - } - - private static class DiscoveryEncodeAddressAsHostname - implements HttpRequestFilter - { - @Override - public Request filterRequest(Request request) - { - return Request.Builder.fromRequest(request) - .setUri(toIpEncodedAsHostnameUri(request.getUri())) - .build(); - } - - private static URI toIpEncodedAsHostnameUri(URI uri) - { - if (!uri.getScheme().equals("https")) { - return uri; - } - try { - String host = uri.getHost(); - InetAddress inetAddress = InetAddress.getByName(host); - String addressAsHostname = encodeAddressAsHostname(inetAddress); - return new URI(uri.getScheme(), uri.getUserInfo(), addressAsHostname, uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); - } - catch (UnknownHostException e) { - throw new UncheckedIOException(e); - } - catch (URISyntaxException e) { - throw new RuntimeException(e); - } - } + install(new InternalCommunicationForDiscoveryModule(ForDiscoveryClient.class)); } } diff --git a/core/trino-main/src/main/java/io/trino/node/AnnounceNodeInventoryModule.java b/core/trino-main/src/main/java/io/trino/node/AnnounceNodeInventoryModule.java index 8fec265da70b..6f7efba7a3bb 100644 --- a/core/trino-main/src/main/java/io/trino/node/AnnounceNodeInventoryModule.java +++ b/core/trino-main/src/main/java/io/trino/node/AnnounceNodeInventoryModule.java @@ -46,5 +46,8 @@ protected void setup(Binder binder) config.setIdleTimeout(new Duration(3, SECONDS)); config.setRequestTimeout(new Duration(3, SECONDS)); }).build()); + + // internal communication setup for discovery http client + install(new InternalCommunicationForDiscoveryModule(ForAnnouncer.class)); } } diff --git a/core/trino-main/src/main/java/io/trino/node/InternalCommunicationForDiscoveryModule.java b/core/trino-main/src/main/java/io/trino/node/InternalCommunicationForDiscoveryModule.java new file mode 100644 index 000000000000..7b47227b11a2 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/node/InternalCommunicationForDiscoveryModule.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.node; + +import com.google.inject.Binder; +import com.google.inject.multibindings.Multibinder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.airlift.http.client.HttpClientConfig; +import io.airlift.http.client.HttpRequestFilter; +import io.airlift.http.client.Request; +import io.trino.server.InternalAuthenticationManager; +import io.trino.server.InternalCommunicationConfig; + +import java.io.UncheckedIOException; +import java.lang.annotation.Annotation; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; + +import static com.google.inject.multibindings.Multibinder.newSetBinder; +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.node.AddressToHostname.encodeAddressAsHostname; +import static io.trino.server.InternalCommunicationHttpClientModule.configureClient; +import static java.util.Objects.requireNonNull; + +public class InternalCommunicationForDiscoveryModule + extends AbstractConfigurationAwareModule +{ + private final Class httpClientQualifier; + + public InternalCommunicationForDiscoveryModule(Class httpClientQualifier) + { + this.httpClientQualifier = requireNonNull(httpClientQualifier, "httpClientQualifier is null"); + } + + @Override + protected void setup(Binder binder) + { + InternalCommunicationConfig internalCommunicationConfig = buildConfigObject(InternalCommunicationConfig.class); + Multibinder discoveryFilterBinder = newSetBinder(binder, HttpRequestFilter.class, httpClientQualifier); + if (internalCommunicationConfig.isHttpsRequired() && internalCommunicationConfig.getKeyStorePath() == null && internalCommunicationConfig.getTrustStorePath() == null) { + discoveryFilterBinder.addBinding().to(DiscoveryEncodeAddressAsHostname.class); + } + configBinder(binder).bindConfigDefaults(HttpClientConfig.class, httpClientQualifier, config -> configureClient(config, internalCommunicationConfig)); + discoveryFilterBinder.addBinding().to(InternalAuthenticationManager.class); + } + + private static class DiscoveryEncodeAddressAsHostname + implements HttpRequestFilter + { + @Override + public Request filterRequest(Request request) + { + return Request.Builder.fromRequest(request) + .setUri(toIpEncodedAsHostnameUri(request.getUri())) + .build(); + } + + private static URI toIpEncodedAsHostnameUri(URI uri) + { + if (!uri.getScheme().equals("https")) { + return uri; + } + try { + String host = uri.getHost(); + InetAddress inetAddress = InetAddress.getByName(host); + String addressAsHostname = encodeAddressAsHostname(inetAddress); + return new URI(uri.getScheme(), uri.getUserInfo(), addressAsHostname, uri.getPort(), uri.getPath(), uri.getQuery(), uri.getFragment()); + } + catch (UnknownHostException e) { + throw new UncheckedIOException(e); + } + catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/operator/AssignUniqueIdOperator.java b/core/trino-main/src/main/java/io/trino/operator/AssignUniqueIdOperator.java index 5c2649e0f688..99cd023c3f56 100644 --- a/core/trino-main/src/main/java/io/trino/operator/AssignUniqueIdOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/AssignUniqueIdOperator.java @@ -123,7 +123,7 @@ private AssignUniqueId(TaskId taskId, AtomicLong rowIdPool) { this.rowIdPool = requireNonNull(rowIdPool, "rowIdPool is null"); - uniqueValueMask = (((long) taskId.getStageId().getId()) << 54) | (((long) taskId.getPartitionId()) << 40); + uniqueValueMask = (((long) taskId.stageId().id()) << 54) | (((long) taskId.partitionId()) << 40); requestValues(); } diff --git a/core/trino-main/src/main/java/io/trino/operator/DeduplicatingDirectExchangeBuffer.java b/core/trino-main/src/main/java/io/trino/operator/DeduplicatingDirectExchangeBuffer.java index e8de58da37a4..113886f53105 100644 --- a/core/trino-main/src/main/java/io/trino/operator/DeduplicatingDirectExchangeBuffer.java +++ b/core/trino-main/src/main/java/io/trino/operator/DeduplicatingDirectExchangeBuffer.java @@ -191,8 +191,8 @@ public synchronized void addTask(TaskId taskId) checkState(!noMoreTasks, "no more tasks expected"); checkState(allTasks.add(taskId), "task already registered: %s", taskId); - if (taskId.getAttemptId() > maxAttemptId) { - maxAttemptId = taskId.getAttemptId(); + if (taskId.attemptId() > maxAttemptId) { + maxAttemptId = taskId.attemptId(); pageBuffer.removePagesForPreviousAttempts(maxAttemptId); updateMaxRetainedSize(); @@ -214,7 +214,7 @@ public synchronized void addPages(TaskId taskId, List pages) checkState(!successfulTasks.contains(taskId), "task is finished: %s", taskId); checkState(!failedTasks.containsKey(taskId), "task is failed: %s", taskId); - if (taskId.getAttemptId() < maxAttemptId) { + if (taskId.attemptId() < maxAttemptId) { return; } @@ -281,7 +281,7 @@ private void checkInputFinished() } Set latestAttemptTasks = allTasks.stream() - .filter(taskId -> taskId.getAttemptId() == maxAttemptId) + .filter(taskId -> taskId.attemptId() == maxAttemptId) .collect(toImmutableSet()); if (successfulTasks.containsAll(latestAttemptTasks)) { @@ -291,7 +291,7 @@ private void checkInputFinished() } Map failures = failedTasks.entrySet().stream() - .filter(entry -> entry.getKey().getAttemptId() == maxAttemptId) + .filter(entry -> entry.getKey().attemptId() == maxAttemptId) .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); Throwable failure = null; @@ -572,9 +572,9 @@ private void writeToSink(TaskId taskId, List pages) updateSinkInstanceHandleIfNecessary(); } } - writeBuffer.writeInt(taskId.getStageId().getId()); - writeBuffer.writeInt(taskId.getPartitionId()); - writeBuffer.writeInt(taskId.getAttemptId()); + writeBuffer.writeInt(taskId.stageId().id()); + writeBuffer.writeInt(taskId.partitionId()); + writeBuffer.writeInt(taskId.attemptId()); writeBuffer.writeBytes(page); exchangeSink.add(0, writeBuffer.slice().copy()); writeBuffer.reset(); @@ -622,7 +622,7 @@ public synchronized void removePagesForPreviousAttempts(int currentAttemptId) while (iterator.hasNext()) { Map.Entry> entry = iterator.next(); TaskId taskId = entry.getKey(); - if (taskId.getAttemptId() < currentAttemptId) { + if (taskId.attemptId() < currentAttemptId) { for (Slice page : entry.getValue()) { removedPagesRetainedSizeInBytes += page.getRetainedSize(); removedPagesCount++; diff --git a/core/trino-main/src/main/java/io/trino/operator/DeleteAndInsertMergeProcessor.java b/core/trino-main/src/main/java/io/trino/operator/DeleteAndInsertMergeProcessor.java index 3dd4b85ab7f5..0813fbf26e5e 100644 --- a/core/trino-main/src/main/java/io/trino/operator/DeleteAndInsertMergeProcessor.java +++ b/core/trino-main/src/main/java/io/trino/operator/DeleteAndInsertMergeProcessor.java @@ -156,13 +156,13 @@ private void addDeleteRow(PageBuilder pageBuilder, Page originalPage, int positi // use a DictionaryBlock to omit columns. // Copy the write redistribution columns for (int targetChannel : dataColumnChannels) { - Type columnType = dataColumnTypes.get(targetChannel); BlockBuilder targetBlock = pageBuilder.getBlockBuilder(targetChannel); int redistributionChannelNumber = redistributionChannelNumbers.get(targetChannel); if (redistributionChannelNumbers.get(targetChannel) >= 0) { // The value comes from that column of the page - columnType.appendTo(originalPage.getBlock(redistributionChannelNumber), position, targetBlock); + Block block = originalPage.getBlock(redistributionChannelNumber); + targetBlock.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } else { // We don't care about the other data columns @@ -177,7 +177,8 @@ private void addDeleteRow(PageBuilder pageBuilder, Page originalPage, int positi INTEGER.writeLong(pageBuilder.getBlockBuilder(dataColumnChannels.size() + 1), -1); // Copy row ID column - rowIdType.appendTo(originalPage.getBlock(rowIdChannel), position, pageBuilder.getBlockBuilder(dataColumnChannels.size() + 2)); + Block rowIdBlock = originalPage.getBlock(rowIdChannel); + pageBuilder.getBlockBuilder(dataColumnChannels.size() + 2).append(rowIdBlock.getUnderlyingValueBlock(), rowIdBlock.getUnderlyingValuePosition(position)); // Write 0, meaning this row is not an insert derived from an update TINYINT.writeLong(pageBuilder.getBlockBuilder(dataColumnChannels.size() + 3), 0); @@ -189,10 +190,10 @@ private void addInsertRow(PageBuilder pageBuilder, List fields, int posit { // Copy the values from the merge block for (int targetChannel : dataColumnChannels) { - Type columnType = dataColumnTypes.get(targetChannel); BlockBuilder targetBlock = pageBuilder.getBlockBuilder(targetChannel); // The value comes from that column of the page - columnType.appendTo(fields.get(targetChannel), position, targetBlock); + Block block = fields.get(targetChannel); + targetBlock.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } // Add the operation column == insert diff --git a/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java b/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java index dd88bdc07e73..8e08cbfd5118 100644 --- a/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/ExchangeOperator.java @@ -96,8 +96,8 @@ public SourceOperator createOperator(DriverContext driverContext) // LazyExchangeDataSource allows to choose an exchange source implementation based on the information received from a split. TaskId taskId = taskContext.getTaskId(); exchangeDataSource = new LazyExchangeDataSource( - taskId.getQueryId(), - new ExchangeId(format("direct-exchange-%s-%s", taskId.getStageId().getId(), sourceId)), + taskId.queryId(), + new ExchangeId(format("direct-exchange-%s-%s", taskId.stageId().id(), sourceId)), taskContext.getSession().getQuerySpan(), directExchangeClientSupplier, memoryContext, diff --git a/core/trino-main/src/main/java/io/trino/operator/ExplainAnalyzeOperator.java b/core/trino-main/src/main/java/io/trino/operator/ExplainAnalyzeOperator.java index 1f383e6400ec..8455c785430e 100644 --- a/core/trino-main/src/main/java/io/trino/operator/ExplainAnalyzeOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/ExplainAnalyzeOperator.java @@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.spi.type.VarcharType.VARCHAR; import static io.trino.sql.planner.planprinter.PlanPrinter.textDistributedPlan; import static java.util.Objects.requireNonNull; @@ -151,17 +150,16 @@ public Page getOutput() return null; } - QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().getQueryId()); + QueryInfo queryInfo = queryPerformanceFetcher.getQueryInfo(operatorContext.getDriverContext().getTaskId().queryId()); checkState(queryInfo.getStages().isPresent(), "Stages informations is missing"); - checkState(queryInfo.getStages().get().getOutputStage().getSubStages().size() == 1, "Expected one sub stage of explain node"); + StagesInfo stagesInfo = queryInfo.getStages().get(); + checkState(stagesInfo.getOutputStage().getSubStages().size() == 1, "Expected one sub stage of explain node"); - if (!hasFinalStageInfo(queryInfo.getStages().get())) { + if (!hasFinalStageInfo(stagesInfo)) { return null; } - List stagesWithoutOutputStage = queryInfo.getStages().orElseThrow().getStages().stream() - .filter(stage -> !stage.getStageId().equals(queryInfo.getStages().orElseThrow().getOutputStageId())) - .collect(toImmutableList()); + List stagesWithoutOutputStage = stagesInfo.getSubStagesDeepTopological(stagesInfo.getOutputStageId(), false); String plan = textDistributedPlan( stagesWithoutOutputStage, @@ -194,7 +192,7 @@ private boolean hasFinalStageInfo(StagesInfo stages) private boolean isFinalStageInfo(StagesInfo stages) { - List subStages = stages.getSubStagesDeepPreOrder(operatorContext.getDriverContext().getTaskId().getStageId()); + List subStages = stages.getSubStagesDeep(operatorContext.getDriverContext().getTaskId().stageId()); return subStages.stream().allMatch(StageInfo::isFinalStageInfo); } } diff --git a/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRankBuilder.java b/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRankBuilder.java index c8c02577c418..282c9ed1efff 100644 --- a/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRankBuilder.java +++ b/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRankBuilder.java @@ -19,6 +19,7 @@ import io.trino.operator.RowReferencePageManager.LoadCursor; import io.trino.spi.Page; import io.trino.spi.PageBuilder; +import io.trino.spi.block.Block; import io.trino.spi.type.Type; import jakarta.annotation.Nullable; @@ -198,7 +199,8 @@ protected Page computeNext() Page page = pageManager.getPage(rowId); int position = pageManager.getPosition(rowId); for (int i = 0; i < sourceTypes.size(); i++) { - sourceTypes.get(i).appendTo(page.getBlock(i), position, pageBuilder.getBlockBuilder(i)); + Block block = page.getBlock(i); + pageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } if (produceRanking) { BIGINT.writeLong(pageBuilder.getBlockBuilder(sourceTypes.size()), rankingOutput.get(currentIndexInGroup)); diff --git a/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRowNumberBuilder.java b/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRowNumberBuilder.java index 7ef0f070c5e7..f94e2bb20ba1 100644 --- a/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRowNumberBuilder.java +++ b/core/trino-main/src/main/java/io/trino/operator/GroupedTopNRowNumberBuilder.java @@ -20,6 +20,7 @@ import io.trino.operator.RowReferencePageManager.LoadCursor; import io.trino.spi.Page; import io.trino.spi.PageBuilder; +import io.trino.spi.block.Block; import io.trino.spi.type.Type; import jakarta.annotation.Nullable; @@ -177,7 +178,8 @@ protected Page computeNext() Page page = pageManager.getPage(rowId); int position = pageManager.getPosition(rowId); for (int i = 0; i < sourceTypes.size(); i++) { - sourceTypes.get(i).appendTo(page.getBlock(i), position, pageBuilder.getBlockBuilder(i)); + Block block = page.getBlock(i); + pageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } if (produceRowNumber) { BIGINT.writeLong(pageBuilder.getBlockBuilder(sourceTypes.size()), currentIndexInGroup + 1); diff --git a/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java b/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java index bd48157736a7..dcaa561e0130 100644 --- a/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/MergeOperator.java @@ -167,8 +167,8 @@ public void addSplit(Split split) TaskContext taskContext = operatorContext.getDriverContext().getPipelineContext().getTaskContext(); DirectExchangeClient client = closer.register(directExchangeClientSupplier.get( - taskContext.getTaskId().getQueryId(), - new ExchangeId(format("direct-exchange-merge-%s-%s", taskContext.getTaskId().getStageId().getId(), sourceId)), + taskContext.getTaskId().queryId(), + new ExchangeId(format("direct-exchange-merge-%s-%s", taskContext.getTaskId().stageId().id(), sourceId)), taskContext.getSession().getQuerySpan(), operatorContext.localUserMemoryContext(), taskContext::sourceTaskFailed, diff --git a/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java b/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java index c5e0c9106edb..18926bd39334 100644 --- a/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java +++ b/core/trino-main/src/main/java/io/trino/operator/OperatorContext.java @@ -350,9 +350,15 @@ private void updatePeakMemoryReservations() // Here, the total memory used to be user+system, and sans revocable. This apparent inconsistency should be removed. // Perhaps, we don't need to track "total memory" here. long totalMemory = userMemory; - peakUserMemoryReservation.accumulateAndGet(userMemory, Math::max); - peakRevocableMemoryReservation.accumulateAndGet(revocableMemory, Math::max); - peakTotalMemoryReservation.accumulateAndGet(totalMemory, Math::max); + if (userMemory > peakUserMemoryReservation.get()) { + peakUserMemoryReservation.accumulateAndGet(userMemory, Math::max); + } + if (revocableMemory > peakRevocableMemoryReservation.get()) { + peakRevocableMemoryReservation.accumulateAndGet(revocableMemory, Math::max); + } + if (totalMemory > peakTotalMemoryReservation.get()) { + peakTotalMemoryReservation.accumulateAndGet(totalMemory, Math::max); + } } public long getReservedRevocableBytes() @@ -522,7 +528,7 @@ public OperatorStats getOperatorStats() long inputPositionsCount = inputPositions.getTotalCount(); return new OperatorStats( - driverContext.getTaskId().getStageId().getId(), + driverContext.getTaskId().stageId().id(), driverContext.getPipelineContext().getPipelineId(), operatorId, planNodeId, diff --git a/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java b/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java index 5f17674ba2c5..e847879cebd0 100644 --- a/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java +++ b/core/trino-main/src/main/java/io/trino/operator/OperatorStats.java @@ -29,7 +29,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Verify.verify; -import static io.trino.execution.DistributionSnapshot.pruneMetrics; import static java.lang.Math.max; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.NANOSECONDS; @@ -636,51 +635,6 @@ private static Mergeable mergeInfos(Mergeable base, List others) return (Mergeable) base.mergeWith(others); } - public OperatorStats pruneDigests() - { - return new OperatorStats( - stageId, - pipelineId, - operatorId, - planNodeId, - sourceId, - operatorType, - totalDrivers, - addInputCalls, - addInputWall, - addInputCpu, - physicalInputDataSize, - physicalInputPositions, - physicalInputReadTime, - internalNetworkInputDataSize, - internalNetworkInputPositions, - inputDataSize, - inputPositions, - sumSquaredInputPositions, - getOutputCalls, - getOutputWall, - getOutputCpu, - outputDataSize, - outputPositions, - dynamicFilterSplitsProcessed, - pruneMetrics(metrics), - pruneMetrics(connectorMetrics), - pruneMetrics(pipelineMetrics), - physicalWrittenDataSize, - blockedWall, - finishCalls, - finishWall, - finishCpu, - userMemoryReservation, - revocableMemoryReservation, - peakUserMemoryReservation, - peakRevocableMemoryReservation, - peakTotalMemoryReservation, - spilledDataSize, - blockedReason, - info); - } - public OperatorStats summarize() { if (info == null || info.isFinal()) { diff --git a/core/trino-main/src/main/java/io/trino/operator/PagesIndex.java b/core/trino-main/src/main/java/io/trino/operator/PagesIndex.java index efcdd0dea929..6a449383b64d 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PagesIndex.java +++ b/core/trino-main/src/main/java/io/trino/operator/PagesIndex.java @@ -316,9 +316,8 @@ private int buildPage(int position, int endPosition, PageBuilder pageBuilder) // append the row pageBuilder.declarePosition(); for (int channel = 0; channel < channels.length; channel++) { - Type type = types.get(channel); Block block = channels[channel].get(blockIndex); - type.appendTo(block, blockPosition, pageBuilder.getBlockBuilder(channel)); + pageBuilder.getBlockBuilder(channel).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(blockPosition)); } position++; @@ -331,10 +330,9 @@ public void appendTo(int channel, int position, BlockBuilder output) { long pageAddress = valueAddresses.getLong(position); - Type type = types.get(channel); Block block = channels[channel].get(decodeSliceIndex(pageAddress)); int blockPosition = decodePosition(pageAddress); - type.appendTo(block, blockPosition, output); + output.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(blockPosition)); } public boolean isNull(int channel, int position) @@ -518,7 +516,7 @@ public PagesSpatialIndexSupplier createPagesSpatialIndex( { // TODO probably shouldn't copy to reduce memory and for memory accounting's sake List> channels = ImmutableList.copyOf(this.channels); - return new PagesSpatialIndexSupplier(session, valueAddresses, types, outputChannels, channels, geometryChannel, radiusChannel, constantRadius, partitionChannel, spatialRelationshipTest, filterFunctionFactory, partitions); + return new PagesSpatialIndexSupplier(session, valueAddresses, outputChannels, channels, geometryChannel, radiusChannel, constantRadius, partitionChannel, spatialRelationshipTest, filterFunctionFactory, partitions); } public LookupSourceSupplier createLookupSourceSupplier( diff --git a/core/trino-main/src/main/java/io/trino/operator/PagesIndexPageSorter.java b/core/trino-main/src/main/java/io/trino/operator/PagesIndexPageSorter.java index 03760a796e49..d4f7e42e300b 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PagesIndexPageSorter.java +++ b/core/trino-main/src/main/java/io/trino/operator/PagesIndexPageSorter.java @@ -19,10 +19,9 @@ import io.trino.spi.connector.SortOrder; import io.trino.spi.type.Type; +import java.util.Iterator; import java.util.List; -import static io.trino.operator.SyntheticAddress.decodePosition; -import static io.trino.operator.SyntheticAddress.decodeSliceIndex; import static java.util.Objects.requireNonNull; public class PagesIndexPageSorter @@ -37,24 +36,12 @@ public PagesIndexPageSorter(PagesIndex.Factory pagesIndexFactory) } @Override - public long[] sort(List types, List pages, List sortChannels, List sortOrders, int expectedPositions) + public Iterator sort(List types, List pages, List sortChannels, List sortOrders, int expectedPositions) { PagesIndex pagesIndex = pagesIndexFactory.newPagesIndex(types, expectedPositions); pages.forEach(pagesIndex::addPage); pagesIndex.sort(sortChannels, sortOrders); - return pagesIndex.getValueAddresses().toLongArray(); - } - - @Override - public int decodePageIndex(long address) - { - return decodeSliceIndex(address); - } - - @Override - public int decodePositionIndex(long address) - { - return decodePosition(address); + return pagesIndex.getSortedPages(); } } diff --git a/core/trino-main/src/main/java/io/trino/operator/PagesRTreeIndex.java b/core/trino-main/src/main/java/io/trino/operator/PagesRTreeIndex.java index de426b58c0ca..e5cbdcbc9c7a 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PagesRTreeIndex.java +++ b/core/trino-main/src/main/java/io/trino/operator/PagesRTreeIndex.java @@ -24,7 +24,6 @@ import io.trino.spi.PageBuilder; import io.trino.spi.block.Block; import io.trino.spi.block.VariableWidthBlock; -import io.trino.spi.type.Type; import io.trino.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory; import it.unimi.dsi.fastutil.ints.IntArrayList; import it.unimi.dsi.fastutil.longs.LongArrayList; @@ -54,7 +53,6 @@ public class PagesRTreeIndex private static final int[] EMPTY_ADDRESSES = new int[0]; private final LongArrayList addresses; - private final List types; private final List outputChannels; private final List> channels; private final STRtree rtree; @@ -103,7 +101,6 @@ public long getEstimatedMemorySizeInBytes() public PagesRTreeIndex( Session session, LongArrayList addresses, - List types, List outputChannels, List> channels, STRtree rtree, @@ -114,7 +111,6 @@ public PagesRTreeIndex( Map partitions) { this.addresses = requireNonNull(addresses, "addresses is null"); - this.types = types; this.outputChannels = outputChannels; this.channels = requireNonNull(channels, "channels is null"); this.rtree = requireNonNull(rtree, "rtree is null"); @@ -229,10 +225,9 @@ public void appendTo(int joinPosition, PageBuilder pageBuilder, int outputChanne int blockPosition = decodePosition(joinAddress); for (int outputIndex : outputChannels) { - Type type = types.get(outputIndex); List channel = channels.get(outputIndex); Block block = channel.get(blockIndex); - type.appendTo(block, blockPosition, pageBuilder.getBlockBuilder(outputChannelOffset)); + pageBuilder.getBlockBuilder(outputChannelOffset).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(blockPosition)); outputChannelOffset++; } } diff --git a/core/trino-main/src/main/java/io/trino/operator/PagesSpatialIndexSupplier.java b/core/trino-main/src/main/java/io/trino/operator/PagesSpatialIndexSupplier.java index c86ddc052027..2a1f8b9ff495 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PagesSpatialIndexSupplier.java +++ b/core/trino-main/src/main/java/io/trino/operator/PagesSpatialIndexSupplier.java @@ -26,7 +26,6 @@ import io.trino.operator.SpatialIndexBuilderOperator.SpatialPredicate; import io.trino.spi.block.Block; import io.trino.spi.block.VariableWidthBlock; -import io.trino.spi.type.Type; import io.trino.sql.gen.JoinFilterFunctionCompiler; import it.unimi.dsi.fastutil.longs.LongArrayList; import it.unimi.dsi.fastutil.objects.ObjectArrayList; @@ -60,7 +59,6 @@ public class PagesSpatialIndexSupplier private final Session session; private final LongArrayList addresses; - private final List types; private final List outputChannels; private final List> channels; private final Optional radiusChannel; @@ -74,7 +72,6 @@ public class PagesSpatialIndexSupplier public PagesSpatialIndexSupplier( Session session, LongArrayList addresses, - List types, List outputChannels, List> channels, int geometryChannel, @@ -87,7 +84,6 @@ public PagesSpatialIndexSupplier( { this.session = session; this.addresses = addresses; - this.types = types; this.outputChannels = outputChannels; this.channels = channels; this.spatialRelationshipTest = spatialRelationshipTest; @@ -109,10 +105,10 @@ private static STRtree buildRTree(LongArrayList addresses, List summarizeOperatorStats(List operatorSummaries) { // Use an exact size ImmutableList builder to avoid a redundant copy in the PipelineStats constructor diff --git a/core/trino-main/src/main/java/io/trino/operator/PipelineStatus.java b/core/trino-main/src/main/java/io/trino/operator/PipelineStatus.java index 5c2801be2972..f2494e900c6a 100644 --- a/core/trino-main/src/main/java/io/trino/operator/PipelineStatus.java +++ b/core/trino-main/src/main/java/io/trino/operator/PipelineStatus.java @@ -13,62 +13,13 @@ */ package io.trino.operator; -import com.google.errorprone.annotations.Immutable; - -@Immutable -public final class PipelineStatus +public record PipelineStatus( + int queuedDrivers, + int runningDrivers, + int blockedDrivers, + int queuedPartitionedDrivers, + long queuedPartitionedSplitsWeight, + int runningPartitionedDrivers, + long runningPartitionedSplitsWeight) { - private final int queuedDrivers; - private final int runningDrivers; - private final int blockedDrivers; - private final int queuedPartitionedDrivers; - private final long queuedPartitionedSplitsWeight; - private final int runningPartitionedDrivers; - private final long runningPartitionedSplitsWeight; - - public PipelineStatus(int queuedDrivers, int runningDrivers, int blockedDrivers, int queuedPartitionedDrivers, long queuedPartitionedSplitsWeight, int runningPartitionedDrivers, long runningPartitionedSplitsWeight) - { - this.queuedDrivers = queuedDrivers; - this.runningDrivers = runningDrivers; - this.blockedDrivers = blockedDrivers; - this.queuedPartitionedDrivers = queuedPartitionedDrivers; - this.queuedPartitionedSplitsWeight = queuedPartitionedSplitsWeight; - this.runningPartitionedDrivers = runningPartitionedDrivers; - this.runningPartitionedSplitsWeight = runningPartitionedSplitsWeight; - } - - public int getQueuedDrivers() - { - return queuedDrivers; - } - - public int getRunningDrivers() - { - return runningDrivers; - } - - public int getBlockedDrivers() - { - return blockedDrivers; - } - - public int getQueuedPartitionedDrivers() - { - return queuedPartitionedDrivers; - } - - public long getQueuedPartitionedSplitsWeight() - { - return queuedPartitionedSplitsWeight; - } - - public int getRunningPartitionedDrivers() - { - return runningPartitionedDrivers; - } - - public long getRunningPartitionedSplitsWeight() - { - return runningPartitionedSplitsWeight; - } } diff --git a/core/trino-main/src/main/java/io/trino/operator/RowNumberOperator.java b/core/trino-main/src/main/java/io/trino/operator/RowNumberOperator.java index 09fbaeb1a03a..61ee4f757100 100644 --- a/core/trino-main/src/main/java/io/trino/operator/RowNumberOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/RowNumberOperator.java @@ -329,8 +329,8 @@ private Page getSelectedRows() pageBuilder.declarePosition(); for (int i = 0; i < outputChannels.length; i++) { int channel = outputChannels[i]; - Type type = types.get(i); - type.appendTo(inputPage.getBlock(channel), currentPosition, pageBuilder.getBlockBuilder(i)); + Block block = inputPage.getBlock(channel); + pageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(currentPosition)); } BIGINT.writeLong(pageBuilder.getBlockBuilder(rowNumberChannel), rowCount + 1); partitionRowCount.set(partitionId, rowCount + 1); diff --git a/core/trino-main/src/main/java/io/trino/operator/SimplePagesHashStrategy.java b/core/trino-main/src/main/java/io/trino/operator/SimplePagesHashStrategy.java index 5ff90f4eec5a..42bcf0ee067b 100644 --- a/core/trino-main/src/main/java/io/trino/operator/SimplePagesHashStrategy.java +++ b/core/trino-main/src/main/java/io/trino/operator/SimplePagesHashStrategy.java @@ -40,7 +40,6 @@ public class SimplePagesHashStrategy implements PagesHashStrategy { private static final int INSTANCE_SIZE = instanceSize(SimplePagesHashStrategy.class); - private final Type[] types; @Nullable private final BlockPositionComparison comparisonOperator; // null when sort channel is absent private final int[] outputChannels; @@ -60,15 +59,14 @@ public SimplePagesHashStrategy( Optional sortChannel, BlockTypeOperators blockTypeOperators) { - this.types = toTypesArray(requireNonNull(types, "types is null")); this.outputChannels = Ints.toArray(requireNonNull(outputChannels, "outputChannels is null")); this.channels = ImmutableList.copyOf(requireNonNull(channels, "channels is null")); checkArgument(types.size() == channels.size(), "Expected types and channels to be the same length"); this.hashChannels = Ints.toArray(requireNonNull(hashChannels, "hashChannels is null")); this.sortChannel = requireNonNull(sortChannel, "sortChannel is null").isEmpty() ? OptionalInt.empty() : OptionalInt.of(sortChannel.get()); - if (this.sortChannel.isPresent() && this.types[this.sortChannel.getAsInt()].isOrderable()) { - this.comparisonOperator = blockTypeOperators.getComparisonUnorderedLastOperator(this.types[this.sortChannel.getAsInt()]); + if (this.sortChannel.isPresent() && types.get(this.sortChannel.getAsInt()).isOrderable()) { + this.comparisonOperator = blockTypeOperators.getComparisonUnorderedLastOperator(types.get(this.sortChannel.getAsInt())); } else { this.comparisonOperator = null; @@ -78,7 +76,7 @@ public SimplePagesHashStrategy( this.hashCodeOperators = new BlockPositionHashCode[this.hashChannels.length]; this.identicalOperators = new BlockPositionIsIdentical[this.hashChannels.length]; for (int i = 0; i < this.hashChannels.length; i++) { - Type type = this.types[this.hashChannels[i]]; + Type type = types.get(this.hashChannels[i]); equalOperators[i] = blockTypeOperators.getEqualOperator(type); hashCodeOperators[i] = blockTypeOperators.getHashCodeOperator(type); identicalOperators[i] = blockTypeOperators.getIdenticalOperator(type); @@ -111,10 +109,9 @@ public long getSizeInBytes() public void appendTo(int blockIndex, int position, PageBuilder pageBuilder, int outputChannelOffset) { for (int outputIndex : outputChannels) { - Type type = types[outputIndex]; List channel = channels.get(outputIndex); Block block = channel.get(blockIndex); - type.appendTo(block, position, pageBuilder.getBlockBuilder(outputChannelOffset)); + pageBuilder.getBlockBuilder(outputChannelOffset).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); outputChannelOffset++; } } diff --git a/core/trino-main/src/main/java/io/trino/operator/SimpleTableExecuteOperator.java b/core/trino-main/src/main/java/io/trino/operator/SimpleTableExecuteOperator.java index e31db7611b28..0d54d991aa7d 100644 --- a/core/trino-main/src/main/java/io/trino/operator/SimpleTableExecuteOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/SimpleTableExecuteOperator.java @@ -13,20 +13,26 @@ */ package io.trino.operator; +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slices; import io.trino.Session; import io.trino.metadata.Metadata; import io.trino.metadata.TableExecuteHandle; import io.trino.spi.Page; +import io.trino.spi.PageBuilder; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.type.Type; import io.trino.sql.planner.plan.PlanNodeId; +import java.util.List; +import java.util.Map; + import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; public class SimpleTableExecuteOperator implements Operator { - private static final Page PAGE = new Page(0); - public static class SimpleTableExecuteOperatorOperatorFactory implements OperatorFactory { @@ -35,6 +41,7 @@ public static class SimpleTableExecuteOperatorOperatorFactory private final Metadata metadata; private final Session session; private final TableExecuteHandle executeHandle; + private final List types; private boolean closed; public SimpleTableExecuteOperatorOperatorFactory( @@ -42,13 +49,15 @@ public SimpleTableExecuteOperatorOperatorFactory( PlanNodeId planNodeId, Metadata metadata, Session session, - TableExecuteHandle executeHandle) + TableExecuteHandle executeHandle, + List types) { this.operatorId = operatorId; this.planNodeId = requireNonNull(planNodeId, "planNodeId is null"); this.metadata = requireNonNull(metadata, "planNodeId is null"); this.session = requireNonNull(session, "planNodeId is null"); this.executeHandle = requireNonNull(executeHandle, "executeHandle is null"); + this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); } @Override @@ -60,7 +69,8 @@ public Operator createOperator(DriverContext driverContext) context, metadata, session, - executeHandle); + executeHandle, + types); } @Override @@ -77,7 +87,8 @@ public OperatorFactory duplicate() planNodeId, metadata, session, - executeHandle); + executeHandle, + types); } } @@ -85,6 +96,7 @@ public OperatorFactory duplicate() private final Metadata metadata; private final Session session; private final TableExecuteHandle executeHandle; + private final List types; private boolean finished; @@ -92,12 +104,14 @@ public SimpleTableExecuteOperator( OperatorContext operatorContext, Metadata metadata, Session session, - TableExecuteHandle executeHandle) + TableExecuteHandle executeHandle, + List types) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.metadata = requireNonNull(metadata, "metadata is null"); this.session = requireNonNull(session, "session is null"); this.executeHandle = requireNonNull(executeHandle, "executeHandle is null"); + this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); } @Override @@ -125,9 +139,17 @@ public Page getOutput() return null; } - metadata.executeTableExecute(session, executeHandle); + Map metrics = metadata.executeTableExecute(session, executeHandle); finished = true; - return PAGE; + PageBuilder pageBuilder = new PageBuilder(types); + BlockBuilder metricNameBuilder = pageBuilder.getBlockBuilder(0); + BlockBuilder metricValueBuilder = pageBuilder.getBlockBuilder(1); + for (Map.Entry entry : metrics.entrySet()) { + types.get(0).writeSlice(metricNameBuilder, Slices.utf8Slice(entry.getKey())); + types.get(1).writeLong(metricValueBuilder, entry.getValue()); + pageBuilder.declarePosition(); + } + return pageBuilder.build(); } @Override diff --git a/core/trino-main/src/main/java/io/trino/operator/SpatialJoinOperator.java b/core/trino-main/src/main/java/io/trino/operator/SpatialJoinOperator.java index 7621e3204a10..44720ce6aa82 100644 --- a/core/trino-main/src/main/java/io/trino/operator/SpatialJoinOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/SpatialJoinOperator.java @@ -136,7 +136,6 @@ public OperatorFactory duplicate() private final OperatorContext operatorContext; private final LocalMemoryContext localUserMemoryContext; private final SpatialJoinNode.Type joinType; - private final List probeTypes; private final List probeOutputChannels; private final int probeGeometryChannel; private final Optional partitionChannel; @@ -173,7 +172,6 @@ public SpatialJoinOperator( this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.localUserMemoryContext = operatorContext.localUserMemoryContext(); this.joinType = requireNonNull(joinType, "joinType is null"); - this.probeTypes = ImmutableList.copyOf(probeTypes); this.probeOutputChannels = ImmutableList.copyOf(probeOutputChannels); this.probeGeometryChannel = probeGeometryChannel; this.partitionChannel = requireNonNull(partitionChannel, "partitionChannel is null"); @@ -302,9 +300,8 @@ private void appendProbe() { int outputChannelOffset = 0; for (int outputIndex : probeOutputChannels) { - Type type = probeTypes.get(outputIndex); Block block = probe.getBlock(outputIndex); - type.appendTo(block, probePosition, pageBuilder.getBlockBuilder(outputChannelOffset)); + pageBuilder.getBlockBuilder(outputChannelOffset).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(probePosition)); outputChannelOffset++; } } diff --git a/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java b/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java index e7249016fbe9..adfc8af784d1 100644 --- a/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/StreamingAggregationOperator.java @@ -313,8 +313,7 @@ private void evaluateAndFlushGroup(Page page, int position) pageBuilder.declarePosition(); for (int i = 0; i < groupByTypes.size(); i++) { Block block = page.getBlock(groupByChannels[i]); - Type type = groupByTypes.get(i); - type.appendTo(block, position, pageBuilder.getBlockBuilder(i)); + pageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } int offset = groupByTypes.size(); for (int i = 0; i < aggregates.size(); i++) { diff --git a/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java b/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java index a16b66075ebd..8733907d2c34 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/TableWriterOperator.java @@ -414,7 +414,9 @@ private void updateMemoryUsage() { long pageSinkMemoryUsage = pageSink.getMemoryUsage(); pageSinkMemoryContext.setBytes(pageSinkMemoryUsage); - pageSinkPeakMemoryUsage.accumulateAndGet(pageSinkMemoryUsage, Math::max); + if (pageSinkMemoryUsage > pageSinkPeakMemoryUsage.get()) { + pageSinkPeakMemoryUsage.accumulateAndGet(pageSinkMemoryUsage, Math::max); + } } @VisibleForTesting diff --git a/core/trino-main/src/main/java/io/trino/operator/TaskContext.java b/core/trino-main/src/main/java/io/trino/operator/TaskContext.java index cc5a29d5c474..0da2e8a3934a 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TaskContext.java +++ b/core/trino-main/src/main/java/io/trino/operator/TaskContext.java @@ -86,7 +86,6 @@ public class TaskContext private final AtomicReference executionStartTime = new AtomicReference<>(); private final AtomicReference lastExecutionStartTime = new AtomicReference<>(); private final AtomicReference terminatingStartTime = new AtomicReference<>(); - private final AtomicReference executionEndTime = new AtomicReference<>(); private final List pipelineContexts = new CopyOnWriteArrayList<>(); @@ -238,9 +237,6 @@ else if (newState.isDone()) { // Only update last start time, if the nothing was started lastExecutionStartTime.compareAndSet(null, now); - // use compare and set from initial value to avoid overwriting if there - // were a duplicate notification, which shouldn't happen - executionEndTime.compareAndSet(null, now); endNanos.compareAndSet(0, nanoTimeNow); endFullGcCount.compareAndSet(-1, majorGcCount); endFullGcTimeNanos.compareAndSet(-1, majorGcTime); @@ -270,8 +266,11 @@ public DataSize getMemoryReservation() public DataSize getPeakMemoryReservation() { long userMemory = taskMemoryContext.getUserMemory(); - currentPeakUserMemoryReservation.updateAndGet(oldValue -> max(oldValue, userMemory)); - return DataSize.ofBytes(currentPeakUserMemoryReservation.get()); + long currentPeakUserMemoryReservation = this.currentPeakUserMemoryReservation.get(); + if (userMemory > currentPeakUserMemoryReservation) { + currentPeakUserMemoryReservation = this.currentPeakUserMemoryReservation.accumulateAndGet(userMemory, Math::max); + } + return DataSize.ofBytes(currentPeakUserMemoryReservation); } public DataSize getRevocableMemoryReservation() @@ -582,7 +581,7 @@ public TaskStats getTaskStats() lastExecutionStartTime.get(), terminatingStartTime.get(), lastExecutionEndTime == 0 ? null : Instant.ofEpochMilli(lastExecutionEndTime), - executionEndTime.get(), + taskStateMachine.getEndTime(), elapsedTime.convertToMostSuccinctTimeUnit(), queuedTime.convertToMostSuccinctTimeUnit(), totalDrivers, diff --git a/core/trino-main/src/main/java/io/trino/operator/TaskStats.java b/core/trino-main/src/main/java/io/trino/operator/TaskStats.java index 1c7ce12c97b5..e4660f1696e0 100644 --- a/core/trino-main/src/main/java/io/trino/operator/TaskStats.java +++ b/core/trino-main/src/main/java/io/trino/operator/TaskStats.java @@ -27,7 +27,6 @@ import java.util.Set; import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -643,55 +642,6 @@ public TaskStats summarizeFinal() summarizePipelineStats(pipelines)); } - public TaskStats pruneDigests() - { - return new TaskStats( - createTime, - firstStartTime, - lastStartTime, - terminatingStartTime, - lastEndTime, - endTime, - elapsedTime, - queuedTime, - totalDrivers, - queuedDrivers, - queuedPartitionedDrivers, - queuedPartitionedSplitsWeight, - runningDrivers, - runningPartitionedDrivers, - runningPartitionedSplitsWeight, - blockedDrivers, - completedDrivers, - cumulativeUserMemory, - userMemoryReservation, - peakUserMemoryReservation, - revocableMemoryReservation, - spilledDataSize, - totalScheduledTime, - totalCpuTime, - totalBlockedTime, - fullyBlocked, - blockedReasons, - physicalInputDataSize, - physicalInputPositions, - physicalInputReadTime, - internalNetworkInputDataSize, - internalNetworkInputPositions, - processedInputDataSize, - processedInputPositions, - inputBlockedTime, - outputDataSize, - outputPositions, - outputBlockedTime, - writerInputDataSize, - physicalWrittenDataSize, - maxWriterCount, - fullGcCount, - fullGcTime, - pipelines.stream().map(PipelineStats::pruneDigests).collect(toImmutableList())); - } - private static List summarizePipelineStats(List pipelines) { // Use an exact size ImmutableList builder to avoid a redundant copy in the TaskStats constructor diff --git a/core/trino-main/src/main/java/io/trino/operator/aggregation/AggregationFromAnnotationsParser.java b/core/trino-main/src/main/java/io/trino/operator/aggregation/AggregationFromAnnotationsParser.java index 4b532d279ece..6f3e81a1bbb5 100644 --- a/core/trino-main/src/main/java/io/trino/operator/aggregation/AggregationFromAnnotationsParser.java +++ b/core/trino-main/src/main/java/io/trino/operator/aggregation/AggregationFromAnnotationsParser.java @@ -34,14 +34,10 @@ import io.trino.spi.function.AggregationState; import io.trino.spi.function.CombineFunction; import io.trino.spi.function.FunctionDependencies; -import io.trino.spi.function.FunctionDependency; import io.trino.spi.function.InOut; import io.trino.spi.function.InputFunction; -import io.trino.spi.function.LiteralParameter; -import io.trino.spi.function.OperatorDependency; import io.trino.spi.function.OutputFunction; import io.trino.spi.function.Signature; -import io.trino.spi.function.TypeParameter; import io.trino.spi.function.WindowAccumulator; import io.trino.spi.type.TypeSignature; @@ -318,10 +314,7 @@ private static IntStream getNonDependencyParameters(Method function) { Annotation[][] parameterAnnotations = function.getParameterAnnotations(); return IntStream.range(0, function.getParameterCount()) - .filter(i -> Arrays.stream(parameterAnnotations[i]).noneMatch(TypeParameter.class::isInstance)) - .filter(i -> Arrays.stream(parameterAnnotations[i]).noneMatch(LiteralParameter.class::isInstance)) - .filter(i -> Arrays.stream(parameterAnnotations[i]).noneMatch(OperatorDependency.class::isInstance)) - .filter(i -> Arrays.stream(parameterAnnotations[i]).noneMatch(FunctionDependency.class::isInstance)); + .filter(i -> Arrays.stream(parameterAnnotations[i]).noneMatch(ImplementationDependency::isImplementationDependencyAnnotation)); } private static List> getNonDependencyParameterTypes(Method function) diff --git a/core/trino-main/src/main/java/io/trino/operator/aggregation/DistinctWindowAccumulator.java b/core/trino-main/src/main/java/io/trino/operator/aggregation/DistinctWindowAccumulator.java index 7334e96f3bb2..59bf70c1d3c5 100644 --- a/core/trino-main/src/main/java/io/trino/operator/aggregation/DistinctWindowAccumulator.java +++ b/core/trino-main/src/main/java/io/trino/operator/aggregation/DistinctWindowAccumulator.java @@ -147,10 +147,8 @@ private void indexCurrentPage(Page page) continue; } for (int channel = 0; channel < argumentChannels.size(); channel++) { - argumentTypes.get(channel).appendTo( - page.getBlock(channel), - position, - filteredPageBuilder.getBlockBuilder(channel)); + Block block = page.getBlock(channel); + filteredPageBuilder.getBlockBuilder(channel).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } filteredPageBuilder.declarePosition(); } diff --git a/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionState.java b/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionState.java index 6faf5c62ddac..60b01bbb2870 100644 --- a/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionState.java +++ b/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionState.java @@ -14,10 +14,8 @@ package io.trino.operator.aggregation.state; import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; import io.trino.spi.function.AccumulatorState; import io.trino.spi.function.AccumulatorStateMetadata; -import io.trino.spi.type.Type; @AccumulatorStateMetadata(stateSerializerClass = BlockPositionStateSerializer.class) public interface BlockPositionState @@ -43,14 +41,4 @@ default void set(BlockPositionState state) setBlock(state.getBlock()); setPosition(state.getPosition()); } - - static void write(Type type, BlockPositionState state, BlockBuilder out) - { - if (state.getBlock() == null) { - out.appendNull(); - } - else { - type.appendTo(state.getBlock(), state.getPosition(), out); - } - } } diff --git a/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionStateSerializer.java b/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionStateSerializer.java index dc683f5f73e7..fe890f17e2e2 100644 --- a/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionStateSerializer.java +++ b/core/trino-main/src/main/java/io/trino/operator/aggregation/state/BlockPositionStateSerializer.java @@ -41,7 +41,8 @@ public void serialize(BlockPositionState state, BlockBuilder out) out.appendNull(); } else { - type.appendTo(state.getBlock(), state.getPosition(), out); + Block block = state.getBlock(); + out.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(state.getPosition())); } } diff --git a/core/trino-main/src/main/java/io/trino/operator/index/IndexSnapshotBuilder.java b/core/trino-main/src/main/java/io/trino/operator/index/IndexSnapshotBuilder.java index 129161625eea..8a2c166d861b 100644 --- a/core/trino-main/src/main/java/io/trino/operator/index/IndexSnapshotBuilder.java +++ b/core/trino-main/src/main/java/io/trino/operator/index/IndexSnapshotBuilder.java @@ -138,8 +138,7 @@ public IndexSnapshot createIndexSnapshot(UnloadedIndexKeyRecordSet indexKeysReco missingKeysPageBuilder.declarePosition(); for (int i = 0; i < page.getChannelCount(); i++) { Block block = page.getBlock(i); - Type type = indexKeysRecordCursor.getType(i); - type.appendTo(block, position, missingKeysPageBuilder.getBlockBuilder(i)); + missingKeysPageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } } } diff --git a/core/trino-main/src/main/java/io/trino/operator/index/StreamingIndexedData.java b/core/trino-main/src/main/java/io/trino/operator/index/StreamingIndexedData.java index 6ebcdedddc64..480114e89572 100644 --- a/core/trino-main/src/main/java/io/trino/operator/index/StreamingIndexedData.java +++ b/core/trino-main/src/main/java/io/trino/operator/index/StreamingIndexedData.java @@ -126,10 +126,9 @@ public void appendTo(long position, PageBuilder pageBuilder, int outputChannelOf checkState(currentPage != null, "getJoinPosition not called first"); int intPosition = toIntExact(position); for (int i = 0; i < outputTypes.size(); i++) { - Type type = outputTypes.get(i); Block block = currentPage.getBlock(i); BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i + outputChannelOffset); - type.appendTo(block, intPosition, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(intPosition)); } } diff --git a/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java b/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java index 8d270c7ef6f1..f5f2ff0c7280 100644 --- a/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java +++ b/core/trino-main/src/main/java/io/trino/operator/join/HashBuilderOperator.java @@ -411,7 +411,8 @@ private ListenableFuture spillIndex() spiller = Optional.of(singleStreamSpillerFactory.create( index.getTypes(), operatorContext.getSpillContext().newLocalSpillContext(), - operatorContext.newLocalUserMemoryContext(HashBuilderOperator.class.getSimpleName()))); + operatorContext.newLocalUserMemoryContext(HashBuilderOperator.class.getSimpleName()), + true)); long spillStartNanos = System.nanoTime(); ListenableFuture spillFuture = getSpiller().spill(index.getPages()); addSuccessCallback(spillFuture, dataSize -> { diff --git a/core/trino-main/src/main/java/io/trino/operator/output/PagePartitioner.java b/core/trino-main/src/main/java/io/trino/operator/output/PagePartitioner.java index cd51332c2bc9..bf896571e460 100644 --- a/core/trino-main/src/main/java/io/trino/operator/output/PagePartitioner.java +++ b/core/trino-main/src/main/java/io/trino/operator/output/PagePartitioner.java @@ -44,6 +44,7 @@ import java.util.function.IntUnaryOperator; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static io.trino.execution.buffer.PageSplitterUtil.splitAndSerializePage; import static io.trino.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; @@ -138,7 +139,16 @@ public void partitionPage(Page page, OperatorContext operatorContext) } int outputPositionCount = replicatesAnyRow && !hasAnyRowBeenReplicated ? page.getPositionCount() + positionsAppenders.length - 1 : page.getPositionCount(); - if (page.getPositionCount() < partitionFunction.partitionCount() * COLUMNAR_STRATEGY_COEFFICIENT) { + if (positionsAppenders.length == 1) { + // single output partition, skip partition calculation and append the entire page to the output partition + checkState(partitionFunction.partitionCount() == 1, "partitionFunction must be single partition"); + // Any output rows are by definition "replicated" when only a single output partition exists + if (replicatesAnyRow && !hasAnyRowBeenReplicated) { + hasAnyRowBeenReplicated = true; + } + positionsAppenders[0].appendToOutputPartition(page); + } + else if (page.getPositionCount() < partitionFunction.partitionCount() * COLUMNAR_STRATEGY_COEFFICIENT) { // Partition will have on average less than COLUMNAR_STRATEGY_COEFFICIENT rows. // Doing it column-wise would degrade performance, so we fall back to row-wise approach. // Performance degradation is the worst in case of skewed hash distribution when only small subset diff --git a/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppender.java b/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppender.java index bc6e4109abc6..b6569004411c 100644 --- a/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppender.java +++ b/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppender.java @@ -14,13 +14,22 @@ package io.trino.operator.output; import io.trino.spi.block.Block; +import io.trino.spi.block.DictionaryBlock; import io.trino.spi.block.ValueBlock; import it.unimi.dsi.fastutil.ints.IntArrayList; -public interface PositionsAppender +public sealed interface PositionsAppender + permits RowPositionsAppender, TypedPositionsAppender { + /** + * Appends the positions from the list, in the specified order. Implementations are not permitted to modify + * the contents of the {@link IntArrayList} argument as it may be passed directly from {@link DictionaryBlock#getRawIds()} + * without a defensive copy. + */ void append(IntArrayList positions, ValueBlock source); + void appendRange(ValueBlock block, int offset, int length); + /** * Appends the specified value positionCount times. * The result is the same as with using {@link PositionsAppender#append(IntArrayList, ValueBlock)} with diff --git a/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppenderPageBuilder.java b/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppenderPageBuilder.java index 91948beec761..ec20e8fc81ce 100644 --- a/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppenderPageBuilder.java +++ b/core/trino-main/src/main/java/io/trino/operator/output/PositionsAppenderPageBuilder.java @@ -27,7 +27,7 @@ import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; -public class PositionsAppenderPageBuilder +public final class PositionsAppenderPageBuilder { private static final int DEFAULT_INITIAL_EXPECTED_ENTRIES = 8; @VisibleForTesting @@ -73,6 +73,17 @@ private PositionsAppenderPageBuilder( } } + public void appendToOutputPartition(Page page) + { + int positionCount = page.getPositionCount(); + declarePositions(positionCount); + + for (int channel = 0; channel < channelAppenders.length; channel++) { + Block block = page.getBlock(channel); + channelAppenders[channel].appendRange(block, 0, positionCount); + } + } + public void appendToOutputPartition(Page page, IntArrayList positions) { declarePositions(positions.size()); diff --git a/core/trino-main/src/main/java/io/trino/operator/output/RowPositionsAppender.java b/core/trino-main/src/main/java/io/trino/operator/output/RowPositionsAppender.java index f40ec4a30ef1..58023f9afc37 100644 --- a/core/trino-main/src/main/java/io/trino/operator/output/RowPositionsAppender.java +++ b/core/trino-main/src/main/java/io/trino/operator/output/RowPositionsAppender.java @@ -31,7 +31,7 @@ import static io.trino.spi.block.RowBlock.fromNotNullSuppressedFieldBlocks; import static java.util.Objects.requireNonNull; -public class RowPositionsAppender +public final class RowPositionsAppender implements PositionsAppender { private static final int INSTANCE_SIZE = instanceSize(RowPositionsAppender.class); @@ -115,6 +115,47 @@ public void append(IntArrayList positions, ValueBlock block) resetSize(); } + @Override + public void appendRange(ValueBlock block, int offset, int length) + { + checkArgument(block instanceof RowBlock, "Block must be instance of %s", RowBlock.class); + if (length == 0) { + return; + } + + RowBlock sourceRowBlock = (RowBlock) block; + ensureCapacity(length); + + Block[] rawFieldBlocks = sourceRowBlock.getRawFieldBlocks(); + int startOffset = sourceRowBlock.getOffsetBase(); + + for (int i = 0; i < fieldAppenders.length; i++) { + fieldAppenders[i].appendRange(rawFieldBlocks[i], startOffset + offset, length); + } + + boolean[] rawRowIsNull = sourceRowBlock.getRawRowIsNull(); + if (rawRowIsNull != null) { + for (int i = 0; i < length; i++) { + boolean isNull = rawRowIsNull[startOffset + offset + i]; + hasNullRow |= isNull; + hasNonNullRow |= !isNull; + if (hasNullRow & hasNonNullRow) { + System.arraycopy(rawRowIsNull, startOffset + offset + i, rowIsNull, positionCount + i, length - i); + break; + } + else { + rowIsNull[positionCount + i] = isNull; + } + } + } + else { + hasNonNullRow = true; + } + + positionCount += length; + resetSize(); + } + @Override public void appendRle(ValueBlock value, int rlePositionCount) { diff --git a/core/trino-main/src/main/java/io/trino/operator/output/TypedPositionsAppender.java b/core/trino-main/src/main/java/io/trino/operator/output/TypedPositionsAppender.java index 141c26811ca6..7f266d6a6909 100644 --- a/core/trino-main/src/main/java/io/trino/operator/output/TypedPositionsAppender.java +++ b/core/trino-main/src/main/java/io/trino/operator/output/TypedPositionsAppender.java @@ -21,7 +21,7 @@ import static io.airlift.slice.SizeOf.instanceSize; -class TypedPositionsAppender +final class TypedPositionsAppender implements PositionsAppender { private static final int INSTANCE_SIZE = instanceSize(TypedPositionsAppender.class); @@ -44,6 +44,12 @@ public void append(IntArrayList positions, ValueBlock block) blockBuilder.appendPositions(block, positions.elements(), 0, positions.size()); } + @Override + public void appendRange(ValueBlock block, int offset, int length) + { + blockBuilder.appendRange(block, offset, length); + } + @Override public void appendRle(ValueBlock block, int count) { diff --git a/core/trino-main/src/main/java/io/trino/operator/output/UnnestingPositionsAppender.java b/core/trino-main/src/main/java/io/trino/operator/output/UnnestingPositionsAppender.java index ed2624687ac9..83957e5d1a77 100644 --- a/core/trino-main/src/main/java/io/trino/operator/output/UnnestingPositionsAppender.java +++ b/core/trino-main/src/main/java/io/trino/operator/output/UnnestingPositionsAppender.java @@ -23,6 +23,7 @@ import it.unimi.dsi.fastutil.ints.IntOpenHashSet; import jakarta.annotation.Nullable; +import java.util.Arrays; import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; @@ -32,12 +33,13 @@ import static io.trino.operator.output.PositionsAppenderUtil.calculateBlockResetSize; import static io.trino.operator.output.PositionsAppenderUtil.calculateNewArraySize; import static java.lang.Math.max; +import static java.util.Objects.checkFromIndexSize; import static java.util.Objects.requireNonNull; /** * Dispatches the {@link #append} and {@link #appendRle} methods to the {@link #delegate} depending on the input {@link Block} class. */ -public class UnnestingPositionsAppender +public final class UnnestingPositionsAppender { private static final int INSTANCE_SIZE = instanceSize(UnnestingPositionsAppender.class); @@ -68,6 +70,50 @@ public UnnestingPositionsAppender(PositionsAppender delegate, Optional { + appendRle(rleBlock.getValue(), length); + } + case DictionaryBlock dictionaryBlock -> { + ValueBlock dictionary = dictionaryBlock.getDictionary(); + if (state == State.UNINITIALIZED) { + state = State.DICTIONARY; + this.dictionary = dictionary; + dictionaryIdsBuilder.appendRange(dictionaryBlock, offset, length); + } + else if (state == State.DICTIONARY && this.dictionary == dictionary) { + dictionaryIdsBuilder.appendRange(dictionaryBlock, offset, length); + } + else { + transitionToDirect(); + + int[] rawIds = dictionaryBlock.getRawIds(); + int rawOffset = dictionaryBlock.getRawIdsOffset(); + checkFromIndexSize(rawOffset + offset, length, rawIds.length); + IntArrayList positionsList; + if (rawOffset + offset == 0) { + // Fast path, no copy necessary + positionsList = IntArrayList.wrap(rawIds, length); + } + else { + positionsList = IntArrayList.wrap(Arrays.copyOfRange(rawIds, rawOffset + offset, rawOffset + offset + length)); + } + delegate.append(positionsList, dictionary); + } + } + case ValueBlock valueBlock -> { + transitionToDirect(); + delegate.appendRange(valueBlock, offset, length); + } + } + } + public void append(IntArrayList positions, Block source) { if (positions.isEmpty()) { @@ -235,7 +281,7 @@ public boolean shouldForceFlushBeforeRelease() return false; } - private static class DictionaryIdsBuilder + private static final class DictionaryIdsBuilder { private static final int INSTANCE_SIZE = instanceSize(DictionaryIdsBuilder.class); @@ -275,6 +321,17 @@ public void appendPositions(IntArrayList positions, DictionaryBlock block) size += positions.size(); } + public void appendRange(DictionaryBlock block, int offset, int length) + { + checkArgument(length > 0, "block has no positions"); + checkFromIndexSize(offset, length, block.getPositionCount()); + ensureCapacity(size + length); + int[] rawIds = block.getRawIds(); + int rawIdsOffset = block.getRawIdsOffset(); + System.arraycopy(rawIds, rawIdsOffset + offset, dictionaryIds, size, length); + size += length; + } + public DictionaryIdsBuilder newBuilderLike() { if (size == 0) { diff --git a/core/trino-main/src/main/java/io/trino/operator/project/MergePages.java b/core/trino-main/src/main/java/io/trino/operator/project/MergePages.java index 7c2d8b5ad43d..2cb59826ddf6 100644 --- a/core/trino-main/src/main/java/io/trino/operator/project/MergePages.java +++ b/core/trino-main/src/main/java/io/trino/operator/project/MergePages.java @@ -21,10 +21,6 @@ import io.trino.spi.Page; import io.trino.spi.PageBuilder; import io.trino.spi.block.Block; -import io.trino.spi.block.BlockBuilder; -import io.trino.spi.block.DictionaryBlock; -import io.trino.spi.block.RunLengthEncodedBlock; -import io.trino.spi.block.ValueBlock; import io.trino.spi.type.Type; import java.util.List; @@ -208,19 +204,8 @@ private void appendPage(Page page) { pageBuilder.declarePositions(page.getPositionCount()); for (int channel = 0; channel < types.size(); channel++) { - appendRawBlock( - page.getBlock(channel), - pageBuilder.getBlockBuilder(channel)); - } - } - - private void appendRawBlock(Block rawBlock, BlockBuilder blockBuilder) - { - int length = rawBlock.getPositionCount(); - switch (rawBlock) { - case RunLengthEncodedBlock rleBlock -> blockBuilder.appendRepeated(rleBlock.getValue(), 0, length); - case DictionaryBlock dictionaryBlock -> blockBuilder.appendPositions(dictionaryBlock.getDictionary(), dictionaryBlock.getRawIds(), dictionaryBlock.getRawIdsOffset(), length); - case ValueBlock valueBlock -> blockBuilder.appendRange(valueBlock, 0, length); + Block rawBlock = page.getBlock(channel); + pageBuilder.getBlockBuilder(channel).appendBlockRange(rawBlock, 0, rawBlock.getPositionCount()); } } diff --git a/core/trino-main/src/main/java/io/trino/operator/project/SelectedPositions.java b/core/trino-main/src/main/java/io/trino/operator/project/SelectedPositions.java index b704ef2df1bc..20008774face 100644 --- a/core/trino-main/src/main/java/io/trino/operator/project/SelectedPositions.java +++ b/core/trino-main/src/main/java/io/trino/operator/project/SelectedPositions.java @@ -13,6 +13,7 @@ */ package io.trino.operator.project; +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkPositionIndexes; import static com.google.common.base.Preconditions.checkState; @@ -62,6 +63,17 @@ public long getRetainedSizeInBytes() return INSTANCE_SIZE + sizeOf(positions); } + @Override + public String toString() + { + return toStringHelper(this) + .add("isList", isList) + .add("positions", positions) + .add("offset", offset) + .add("size", size) + .toString(); + } + public boolean isList() { return isList; diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatFunction.java index 91d154b250f9..7093b43cf4db 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatFunction.java @@ -22,7 +22,6 @@ import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.Signature; import io.trino.spi.type.ArrayType; -import io.trino.spi.type.Type; import io.trino.spi.type.TypeSignature; import io.trino.sql.gen.VarArgsToArrayAdapterGenerator; @@ -53,7 +52,7 @@ public final class ArrayConcatFunction static { try { MethodHandles.Lookup lookup = lookup(); - METHOD_HANDLE = lookup.findStatic(ArrayConcatFunction.class, "concat", methodType(Block.class, Type.class, Object.class, Block[].class)); + METHOD_HANDLE = lookup.findStatic(ArrayConcatFunction.class, "concat", methodType(Block.class, Object.class, Block[].class)); USER_STATE_FACTORY = lookup.findStatic(ArrayConcatFunction.class, "createState", methodType(Object.class, ArrayType.class)); } catch (ReflectiveOperationException e) { @@ -87,7 +86,7 @@ protected SpecializedSqlScalarFunction specialize(BoundSignature boundSignature) Block.class, Block.class, boundSignature.getArity(), - METHOD_HANDLE.bindTo(arrayType.getElementType()), + METHOD_HANDLE, USER_STATE_FACTORY.bindTo(arrayType)); return new ChoicesSpecializedSqlScalarFunction( @@ -105,7 +104,7 @@ public static Object createState(ArrayType arrayType) } @UsedByGeneratedCode - public static Block concat(Type elementType, Object state, Block[] blocks) + public static Block concat(Object state, Block[] blocks) { int resultPositionCount = 0; @@ -126,9 +125,7 @@ public static Block concat(Type elementType, Object state, Block[] blocks) return ((BufferedArrayValueBuilder) state).build(resultPositionCount, elementBuilder -> { for (Block block : blocks) { - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, elementBuilder); - } + elementBuilder.appendBlockRange(block, 0, block.getPositionCount()); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatUtils.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatUtils.java index b5b27aeccaa0..633412f873e4 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatUtils.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayConcatUtils.java @@ -27,9 +27,7 @@ private ArrayConcatUtils() {} public static Block appendElement(Type elementType, Block block, long value) { BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); elementType.writeLong(blockBuilder, value); @@ -40,9 +38,7 @@ public static Block appendElement(Type elementType, Block block, long value) public static Block appendElement(Type elementType, Block block, boolean value) { BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); elementType.writeBoolean(blockBuilder, value); @@ -53,9 +49,7 @@ public static Block appendElement(Type elementType, Block block, boolean value) public static Block appendElement(Type elementType, Block block, double value) { BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); elementType.writeDouble(blockBuilder, value); @@ -66,9 +60,7 @@ public static Block appendElement(Type elementType, Block block, double value) public static Block appendElement(Type elementType, Block block, Slice value) { BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); elementType.writeSlice(blockBuilder, value); @@ -79,9 +71,7 @@ public static Block appendElement(Type elementType, Block block, Slice value) public static Block appendElement(Type elementType, Block block, Object value) { BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); elementType.writeObject(blockBuilder, value); @@ -95,9 +85,7 @@ public static Block prependElement(Type elementType, Slice value, Block block) BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); elementType.writeSlice(blockBuilder, value); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); return blockBuilder.build(); } @@ -108,9 +96,7 @@ public static Block prependElement(Type elementType, Object value, Block block) BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); elementType.writeObject(blockBuilder, value); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); return blockBuilder.build(); } @@ -121,9 +107,7 @@ public static Block prependElement(Type elementType, long value, Block block) BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); elementType.writeLong(blockBuilder, value); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); return blockBuilder.build(); } @@ -134,9 +118,7 @@ public static Block prependElement(Type elementType, boolean value, Block block) BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); elementType.writeBoolean(blockBuilder, value); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); return blockBuilder.build(); } @@ -147,9 +129,7 @@ public static Block prependElement(Type elementType, double value, Block block) BlockBuilder blockBuilder = elementType.createBlockBuilder(null, block.getPositionCount() + 1); elementType.writeDouble(blockBuilder, value); - for (int i = 0; i < block.getPositionCount(); i++) { - elementType.appendTo(block, i, blockBuilder); - } + blockBuilder.appendBlockRange(block, 0, block.getPositionCount()); return blockBuilder.build(); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayDistinctFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayDistinctFunction.java index fbb63b42990a..97fd88e79945 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayDistinctFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayDistinctFunction.java @@ -51,7 +51,6 @@ public ArrayDistinctFunction(@TypeParameter("E") Type elementType) @TypeParameter("E") @SqlType("array(E)") public Block distinct( - @TypeParameter("E") Type type, @OperatorDependency( operator = IDENTICAL, argumentTypes = {"E", "E"}, @@ -75,7 +74,6 @@ public Block distinct( } BlockSet distinctElements = new BlockSet( - type, elementIdentical, elementHashCode, array.getPositionCount()); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayExceptFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayExceptFunction.java index 9616f806c7a4..a54b06396e60 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayExceptFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayExceptFunction.java @@ -15,6 +15,7 @@ import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.Convention; import io.trino.spi.function.Description; import io.trino.spi.function.OperatorDependency; @@ -58,14 +59,16 @@ public static Block except( return leftArray; } - BlockSet set = new BlockSet(type, identicalOperator, elementHashCode, rightPositionCount + leftPositionCount); + BlockSet set = new BlockSet(identicalOperator, elementHashCode, rightPositionCount + leftPositionCount); for (int i = 0; i < rightPositionCount; i++) { set.add(rightArray, i); } BlockBuilder distinctElementBlockBuilder = type.createBlockBuilder(null, leftPositionCount); + ValueBlock leftValueBlock = leftArray.getUnderlyingValueBlock(); for (int i = 0; i < leftPositionCount; i++) { - if (set.add(leftArray, i)) { - type.appendTo(leftArray, i, distinctElementBlockBuilder); + int leftPosition = leftArray.getUnderlyingValuePosition(i); + if (set.add(leftValueBlock, leftPosition)) { + distinctElementBlockBuilder.append(leftValueBlock, leftPosition); } } return distinctElementBlockBuilder.build(); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayFlattenFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayFlattenFunction.java index 811908fadf56..6ce63a65ec5e 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayFlattenFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayFlattenFunction.java @@ -75,9 +75,7 @@ public static Block flatten(Type type, Type arrayType, Block array) for (int i = 0; i < array.getPositionCount(); i++) { if (!array.isNull(i)) { Block subArray = (Block) arrayType.getObject(array, i); - for (int j = 0; j < subArray.getPositionCount(); j++) { - type.appendTo(subArray, j, builder); - } + builder.appendBlockRange(subArray, 0, subArray.getPositionCount()); } } return builder.build(); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayIntersectFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayIntersectFunction.java index af451a7d970e..52aecb88f4db 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayIntersectFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayIntersectFunction.java @@ -47,7 +47,6 @@ public ArrayIntersectFunction(@TypeParameter("E") Type elementType) @TypeParameter("E") @SqlType("array(E)") public Block intersect( - @TypeParameter("E") Type type, @OperatorDependency( operator = IDENTICAL, argumentTypes = {"E", "E"}, @@ -72,13 +71,13 @@ public Block intersect( return rightArray; } - BlockSet rightSet = new BlockSet(type, elementIdentical, elementHashCode, rightPositionCount); + BlockSet rightSet = new BlockSet(elementIdentical, elementHashCode, rightPositionCount); for (int i = 0; i < rightPositionCount; i++) { rightSet.add(rightArray, i); } // The intersected set can have at most rightPositionCount elements - BlockSet intersectSet = new BlockSet(type, elementIdentical, elementHashCode, rightSet.size()); + BlockSet intersectSet = new BlockSet(elementIdentical, elementHashCode, rightSet.size()); for (int i = 0; i < leftPositionCount; i++) { if (rightSet.contains(leftArray, i)) { intersectSet.add(leftArray, i); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayRemoveFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayRemoveFunction.java index 879a6c86e1ca..5bf1c193c22c 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayRemoveFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayRemoveFunction.java @@ -16,6 +16,7 @@ import io.trino.spi.TrinoException; import io.trino.spi.block.Block; import io.trino.spi.block.BufferedArrayValueBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.Convention; import io.trino.spi.function.Description; import io.trino.spi.function.OperatorDependency; @@ -88,8 +89,9 @@ public Block remove( } return arrayValueBuilder.build(positions.size(), elementBuilder -> { + ValueBlock valueBlock = array.getUnderlyingValueBlock(); for (int position : positions) { - type.appendTo(array, position, elementBuilder); + elementBuilder.append(valueBlock, array.getUnderlyingValuePosition(position)); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayReverseFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayReverseFunction.java index 365e1b5a44e8..d74574cd64ab 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayReverseFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayReverseFunction.java @@ -15,6 +15,7 @@ import io.trino.spi.block.Block; import io.trino.spi.block.BufferedArrayValueBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.Description; import io.trino.spi.function.ScalarFunction; import io.trino.spi.function.SqlType; @@ -36,9 +37,7 @@ public ArrayReverseFunction(@TypeParameter("E") Type elementType) @TypeParameter("E") @SqlType("array(E)") - public Block reverse( - @TypeParameter("E") Type type, - @SqlType("array(E)") Block block) + public Block reverse(@SqlType("array(E)") Block block) { int arrayLength = block.getPositionCount(); @@ -47,8 +46,9 @@ public Block reverse( } return arrayValueBuilder.build(arrayLength, elementBuilder -> { + ValueBlock valueBlock = block.getUnderlyingValueBlock(); for (int i = arrayLength - 1; i >= 0; i--) { - type.appendTo(block, i, elementBuilder); + elementBuilder.append(valueBlock, block.getUnderlyingValuePosition(i)); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayShuffleFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayShuffleFunction.java index ffce6ef37a5c..a954134bfb39 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayShuffleFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayShuffleFunction.java @@ -15,6 +15,7 @@ import io.trino.spi.block.Block; import io.trino.spi.block.BufferedArrayValueBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.Description; import io.trino.spi.function.ScalarFunction; import io.trino.spi.function.SqlType; @@ -40,9 +41,7 @@ public ArrayShuffleFunction(@TypeParameter("E") Type elementType) @TypeParameter("E") @SqlType("array(E)") - public Block shuffle( - @TypeParameter("E") Type type, - @SqlType("array(E)") Block block) + public Block shuffle(@SqlType("array(E)") Block block) { int length = block.getPositionCount(); if (positions.length < length) { @@ -62,8 +61,9 @@ public Block shuffle( } return arrayValueBuilder.build(length, elementBuilder -> { + ValueBlock valueBlock = block.getUnderlyingValueBlock(); for (int i = 0; i < length; i++) { - type.appendTo(block, positions[i], elementBuilder); + elementBuilder.append(valueBlock, block.getUnderlyingValuePosition(positions[i])); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortComparatorFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortComparatorFunction.java index b27fca5b36c1..9f90e27003c2 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortComparatorFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortComparatorFunction.java @@ -17,6 +17,7 @@ import io.trino.spi.TrinoException; import io.trino.spi.block.Block; import io.trino.spi.block.BufferedArrayValueBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.Convention; import io.trino.spi.function.Description; import io.trino.spi.function.OperatorDependency; @@ -55,7 +56,6 @@ public ArraySortComparatorFunction(@TypeParameter("T") Type elementType) @TypeParameter("T") @SqlType("array(T)") public Block sort( - @TypeParameter("T") Type type, @OperatorDependency(operator = READ_VALUE, argumentTypes = "T", convention = @Convention(arguments = BLOCK_POSITION_NOT_NULL, result = FAIL_ON_NULL)) MethodHandle readValue, @SqlType("array(T)") Block block, @SqlType("function(T, T, integer)") ComparatorObjectLambda function) @@ -78,8 +78,9 @@ public Block sort( sortPositions(arrayLength, comparator); return arrayValueBuilder.build(arrayLength, elementBuilder -> { + ValueBlock valueBlock = block.getUnderlyingValueBlock(); for (int i = 0; i < arrayLength; i++) { - type.appendTo(block, positions.get(i), elementBuilder); + elementBuilder.append(valueBlock, block.getUnderlyingValuePosition(positions.get(i))); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortFunction.java index 4a279f1268ad..2bb7cb771ba3 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArraySortFunction.java @@ -15,6 +15,7 @@ import io.trino.spi.block.Block; import io.trino.spi.block.BufferedArrayValueBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.Convention; import io.trino.spi.function.Description; import io.trino.spi.function.OperatorDependency; @@ -52,7 +53,6 @@ public Block sort( operator = COMPARISON_UNORDERED_LAST, argumentTypes = {"E", "E"}, convention = @Convention(arguments = {BLOCK_POSITION_NOT_NULL, BLOCK_POSITION_NOT_NULL}, result = FAIL_ON_NULL)) BlockPositionComparison comparisonOperator, - @TypeParameter("E") Type type, @SqlType("array(E)") Block block) { int arrayLength = block.getPositionCount(); @@ -78,8 +78,9 @@ public Block sort( }); return arrayValueBuilder.build(arrayLength, elementBuilder -> { + ValueBlock valueBlock = block.getUnderlyingValueBlock(); for (int i = 0; i < arrayLength; i++) { - type.appendTo(block, positions.getInt(i), elementBuilder); + elementBuilder.append(valueBlock, block.getUnderlyingValuePosition(positions.getInt(i))); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayUnionFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayUnionFunction.java index 13667147d223..0d672bc33828 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayUnionFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArrayUnionFunction.java @@ -58,7 +58,6 @@ public static Block union( @SqlType("array(E)") Block rightArray) { BlockSet set = new BlockSet( - type, elementIdentical, elementHashCode, leftArray.getPositionCount() + rightArray.getPositionCount()); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ArraysOverlapFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ArraysOverlapFunction.java index 52d0ccc46caa..baf40d69b9f2 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ArraysOverlapFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ArraysOverlapFunction.java @@ -22,7 +22,6 @@ import io.trino.spi.function.SqlType; import io.trino.spi.function.TypeParameter; import io.trino.spi.type.StandardTypes; -import io.trino.spi.type.Type; import io.trino.type.BlockTypeOperators; import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.BLOCK_POSITION; @@ -40,7 +39,6 @@ private ArraysOverlapFunction() {} @TypeParameter("E") @SqlType(StandardTypes.BOOLEAN) public static Boolean arraysOverlap( - @TypeParameter("E") Type type, @OperatorDependency( operator = IDENTICAL, argumentTypes = {"E", "E"}, @@ -67,7 +65,7 @@ public static Boolean arraysOverlap( return false; } - BlockSet smallerSet = new BlockSet(type, elementIdentical, elementHashCode, smallerPositionCount); + BlockSet smallerSet = new BlockSet(elementIdentical, elementHashCode, smallerPositionCount); for (int position = 0; position < smallerPositionCount; position++) { smallerSet.add(smaller, position); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/BlockSet.java b/core/trino-main/src/main/java/io/trino/operator/scalar/BlockSet.java index 5f6c022ae45d..035d218c0554 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/BlockSet.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/BlockSet.java @@ -17,7 +17,6 @@ import io.trino.spi.TrinoException; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; -import io.trino.spi.type.Type; import io.trino.type.BlockTypeOperators.BlockPositionHashCode; import io.trino.type.BlockTypeOperators.BlockPositionIsIdentical; @@ -47,7 +46,6 @@ public class BlockSet private static final float FILL_RATIO = 0.75f; private static final int EMPTY_SLOT = -1; - private final Type elementType; private final BlockPositionIsIdentical elementIdenticalOperator; private final BlockPositionHashCode elementHashCodeOperator; @@ -64,13 +62,11 @@ public class BlockSet private boolean containsNullElement; public BlockSet( - Type elementType, BlockPositionIsIdentical elementIdenticalOperator, BlockPositionHashCode elementHashCodeOperator, int maximumSize) { checkArgument(maximumSize >= 0, "maximumSize must not be negative"); - this.elementType = requireNonNull(elementType, "elementType is null"); this.elementIdenticalOperator = requireNonNull(elementIdenticalOperator, "elementIdenticalOperator is null"); this.elementHashCodeOperator = requireNonNull(elementHashCodeOperator, "elementHashCodeOperator is null"); this.maximumSize = maximumSize; @@ -163,7 +159,7 @@ public void getAllWithSizeLimit(BlockBuilder blockBuilder, String functionName, long initialSize = blockBuilder.getSizeInBytes(); long maxBlockMemoryInBytes = toIntExact(maxFunctionMemory.toBytes()); for (int i = 0; i < size; i++) { - elementType.appendTo(elementBlocks[i], elementPositions[i], blockBuilder); + blockBuilder.append(elementBlocks[i].getUnderlyingValueBlock(), elementBlocks[i].getUnderlyingValuePosition(elementPositions[i])); if (blockBuilder.getSizeInBytes() - initialSize > maxBlockMemoryInBytes) { throw new TrinoException( EXCEEDED_FUNCTION_MEMORY_LIMIT, diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MapConcatFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MapConcatFunction.java index 9acc511ad945..c7a629f1753a 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MapConcatFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MapConcatFunction.java @@ -55,7 +55,6 @@ public final class MapConcatFunction private static final MethodHandle METHOD_HANDLE = methodHandle( MapConcatFunction.class, "mapConcat", - MapType.class, BlockPositionIsIdentical.class, BlockPositionHashCode.class, Object.class, @@ -94,7 +93,7 @@ protected SpecializedSqlScalarFunction specialize(BoundSignature boundSignature) SqlMap.class, SqlMap.class, boundSignature.getArity(), - MethodHandles.insertArguments(METHOD_HANDLE, 0, mapType, keysIdenticalOperator, keyHashCode), + MethodHandles.insertArguments(METHOD_HANDLE, 0, keysIdenticalOperator, keyHashCode), USER_STATE_FACTORY.bindTo(mapType)); return new ChoicesSpecializedSqlScalarFunction( @@ -112,7 +111,7 @@ public static Object createMapState(MapType mapType) } @UsedByGeneratedCode - public static SqlMap mapConcat(MapType mapType, BlockPositionIsIdentical keysIdenticalOperator, BlockPositionHashCode keyHashCode, Object state, SqlMap[] maps) + public static SqlMap mapConcat(BlockPositionIsIdentical keysIdenticalOperator, BlockPositionHashCode keyHashCode, Object state, SqlMap[] maps) { int maxEntries = 0; int lastMapIndex = maps.length - 1; @@ -133,9 +132,7 @@ public static SqlMap mapConcat(MapType mapType, BlockPositionIsIdentical keysIde BufferedMapValueBuilder mapValueBuilder = (BufferedMapValueBuilder) state; - Type keyType = mapType.getKeyType(); - Type valueType = mapType.getValueType(); - BlockSet set = new BlockSet(keyType, keysIdenticalOperator, keyHashCode, maxEntries); + BlockSet set = new BlockSet(keysIdenticalOperator, keyHashCode, maxEntries); return mapValueBuilder.build(maxEntries, (keyBuilder, valueBuilder) -> { // the last map SqlMap map = maps[last]; @@ -144,7 +141,7 @@ public static SqlMap mapConcat(MapType mapType, BlockPositionIsIdentical keysIde Block rawValueBlock = map.getRawValueBlock(); for (int i = 0; i < map.getSize(); i++) { set.add(rawKeyBlock, rawOffset + i); - writeEntry(keyType, valueType, keyBuilder, valueBuilder, rawKeyBlock, rawValueBlock, rawOffset + i); + writeEntry(keyBuilder, valueBuilder, rawKeyBlock, rawValueBlock, rawOffset + i); } // the map between the last and the first @@ -155,7 +152,7 @@ public static SqlMap mapConcat(MapType mapType, BlockPositionIsIdentical keysIde rawValueBlock = map.getRawValueBlock(); for (int i = 0; i < map.getSize(); i++) { if (set.add(rawKeyBlock, rawOffset + i)) { - writeEntry(keyType, valueType, keyBuilder, valueBuilder, rawKeyBlock, rawValueBlock, rawOffset + i); + writeEntry(keyBuilder, valueBuilder, rawKeyBlock, rawValueBlock, rawOffset + i); } } } @@ -167,15 +164,15 @@ public static SqlMap mapConcat(MapType mapType, BlockPositionIsIdentical keysIde rawValueBlock = map.getRawValueBlock(); for (int i = 0; i < map.getSize(); i++) { if (!set.contains(rawKeyBlock, rawOffset + i)) { - writeEntry(keyType, valueType, keyBuilder, valueBuilder, rawKeyBlock, rawValueBlock, rawOffset + i); + writeEntry(keyBuilder, valueBuilder, rawKeyBlock, rawValueBlock, rawOffset + i); } } }); } - private static void writeEntry(Type keyType, Type valueType, BlockBuilder keyBuilder, BlockBuilder valueBuilder, Block rawKeyBlock, Block rawValueBlock, int rawIndex) + private static void writeEntry(BlockBuilder keyBuilder, BlockBuilder valueBuilder, Block rawKeyBlock, Block rawValueBlock, int rawIndex) { - keyType.appendTo(rawKeyBlock, rawIndex, keyBuilder); - valueType.appendTo(rawValueBlock, rawIndex, valueBuilder); + keyBuilder.append(rawKeyBlock.getUnderlyingValueBlock(), rawKeyBlock.getUnderlyingValuePosition(rawIndex)); + valueBuilder.append(rawValueBlock.getUnderlyingValueBlock(), rawValueBlock.getUnderlyingValuePosition(rawIndex)); } } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MapEntriesFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MapEntriesFunction.java index e1a1272b9f11..cd08d9cf70c5 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MapEntriesFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MapEntriesFunction.java @@ -17,6 +17,7 @@ import io.trino.spi.block.BufferedArrayValueBuilder; import io.trino.spi.block.RowBlockBuilder; import io.trino.spi.block.SqlMap; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.Description; import io.trino.spi.function.ScalarFunction; import io.trino.spi.function.SqlType; @@ -49,20 +50,19 @@ public Block mapFromEntries( { verify(rowType.getTypeParameters().size() == 2); - Type keyType = rowType.getTypeParameters().get(0); - Type valueType = rowType.getTypeParameters().get(1); - int size = sqlMap.getSize(); int rawOffset = sqlMap.getRawOffset(); Block rawKeyBlock = sqlMap.getRawKeyBlock(); Block rawValueBlock = sqlMap.getRawValueBlock(); return arrayValueBuilder.build(size, valueBuilder -> { + ValueBlock keyBlock = rawKeyBlock.getUnderlyingValueBlock(); + ValueBlock valueBlock = rawValueBlock.getUnderlyingValueBlock(); for (int i = 0; i < size; i++) { int offset = rawOffset + i; ((RowBlockBuilder) valueBuilder).buildEntry(fieldBuilders -> { - keyType.appendTo(rawKeyBlock, offset, fieldBuilders.get(0)); - valueType.appendTo(rawValueBlock, offset, fieldBuilders.get(1)); + fieldBuilders.get(0).append(keyBlock, rawKeyBlock.getUnderlyingValuePosition(offset)); + fieldBuilders.get(1).append(valueBlock, rawValueBlock.getUnderlyingValuePosition(offset)); }); } }); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MapFilterFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MapFilterFunction.java index 5d260e973eba..e73d02b6a0b0 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MapFilterFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MapFilterFunction.java @@ -32,6 +32,7 @@ import io.trino.spi.block.BufferedMapValueBuilder; import io.trino.spi.block.MapValueBuilder; import io.trino.spi.block.SqlMap; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.BoundSignature; import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.Signature; @@ -206,8 +207,16 @@ private static MethodDefinition generateFilterInner(ClassDefinition definition, .append(new IfStatement("if (keep != null && keep) ...") .condition(and(notEqual(keep, constantNull(Boolean.class)), keep.cast(boolean.class))) .ifTrue(new BytecodeBlock() - .append(keySqlType.invoke("appendTo", void.class, rawKeyBlock, add(index, rawOffset), keyBuilder)) - .append(valueSqlType.invoke("appendTo", void.class, rawValueBlock, add(index, rawOffset), valueBuilder)))))); + .append(keyBuilder.invoke( + "append", + void.class, + rawKeyBlock.invoke("getUnderlyingValueBlock", ValueBlock.class), + rawKeyBlock.invoke("getUnderlyingValuePosition", int.class, add(index, rawOffset)))) + .append(valueBuilder.invoke( + "append", + void.class, + rawValueBlock.invoke("getUnderlyingValueBlock", ValueBlock.class), + rawValueBlock.invoke("getUnderlyingValuePosition", int.class, add(index, rawOffset)))))))); body.ret(); return method; diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MapFromEntriesFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MapFromEntriesFunction.java index 53a7f612f20a..5cf4a0aa48b1 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MapFromEntriesFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MapFromEntriesFunction.java @@ -89,9 +89,10 @@ public SqlMap mapFromEntries( if (keyBlock.isNull(rawIndex)) { throw new TrinoException(INVALID_FUNCTION_ARGUMENT, "map key cannot be null"); } - keyType.appendTo(keyBlock, rawIndex, keyBuilder); + keyBuilder.append(keyBlock.getUnderlyingValueBlock(), keyBlock.getUnderlyingValuePosition(rawIndex)); - valueType.appendTo(entry.getRawFieldBlock(1), rawIndex, valueBuilder); + Block rawValueBlock = entry.getRawFieldBlock(1); + valueBuilder.append(rawValueBlock.getUnderlyingValueBlock(), rawValueBlock.getUnderlyingValuePosition(rawIndex)); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformKeysFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformKeysFunction.java index 6cd275aaf514..71f5eff6924e 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformKeysFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformKeysFunction.java @@ -36,6 +36,7 @@ import io.trino.spi.block.DuplicateMapKeyException; import io.trino.spi.block.MapValueBuilder; import io.trino.spi.block.SqlMap; +import io.trino.spi.block.ValueBlock; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.function.BoundSignature; import io.trino.spi.function.FunctionMetadata; @@ -243,7 +244,11 @@ private static MethodDefinition generateTransformKeyInner(ClassDefinition defini .ifTrue(throwNullKeyException) .ifFalse(new BytecodeBlock() .append(constantType(binder, transformedKeyType).writeValue(keyBuilder, transformedKeyElement.cast(transformedKeyType.getJavaType()))) - .append(valueSqlType.invoke("appendTo", void.class, rawValueBlock, add(index, rawOffset), valueBuilder)))); + .append(valueBuilder.invoke( + "append", + void.class, + rawValueBlock.invoke("getUnderlyingValueBlock", ValueBlock.class), + rawValueBlock.invoke("getUnderlyingValuePosition", int.class, add(index, rawOffset)))))); } else { // key cannot be unknown diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformValuesFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformValuesFunction.java index d367f7afa875..62adc37eb1cf 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformValuesFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MapTransformValuesFunction.java @@ -36,6 +36,7 @@ import io.trino.spi.block.BufferedMapValueBuilder; import io.trino.spi.block.MapValueBuilder; import io.trino.spi.block.SqlMap; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.BoundSignature; import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.Signature; @@ -253,7 +254,11 @@ private static MethodDefinition generateTransformInner(ClassDefinition definitio .append(newInstance(RuntimeException.class, transformationException)) .throwObject(), ImmutableList.of(type(Throwable.class)))))) - .append(keySqlType.invoke("appendTo", void.class, rawKeyBlock, add(index, rawOffset), keyBuilder)) + .append(keyBuilder.invoke( + "append", + void.class, + rawKeyBlock.invoke("getUnderlyingValueBlock", ValueBlock.class), + rawKeyBlock.invoke("getUnderlyingValuePosition", int.class, add(index, rawOffset)))) .append(writeTransformedValueElement))); body.ret(); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MapZipWithFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MapZipWithFunction.java index d32abb0f1749..159ef03ac1bc 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MapZipWithFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MapZipWithFunction.java @@ -18,6 +18,7 @@ import io.trino.spi.block.Block; import io.trino.spi.block.BufferedMapValueBuilder; import io.trino.spi.block.SqlMap; +import io.trino.spi.block.ValueBlock; import io.trino.spi.function.BoundSignature; import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.Signature; @@ -111,6 +112,7 @@ public static SqlMap mapZipWith( return mapValueBuilder.build(maxOutputSize, (keyBuilder, valueBuilder) -> { // seekKey() can take non-trivial time when key is a complicated value, such as a long VARCHAR or ROW. boolean[] keyFound = new boolean[rightSize]; + ValueBlock leftKeyBlock = leftRawKeyBlock.getUnderlyingValueBlock(); for (int leftIndex = 0; leftIndex < leftSize; leftIndex++) { Object key = readNativeValue(keyType, leftRawKeyBlock, leftRawOffset + leftIndex); Object leftValue = readNativeValue(leftValueType, leftRawValueBlock, leftRawOffset + leftIndex); @@ -124,11 +126,12 @@ public static SqlMap mapZipWith( Object outputValue = function.apply(key, leftValue, rightValue); - keyType.appendTo(leftRawKeyBlock, leftRawOffset + leftIndex, keyBuilder); + keyBuilder.append(leftKeyBlock, leftRawKeyBlock.getUnderlyingValuePosition(leftRawOffset + leftIndex)); writeNativeValue(outputValueType, valueBuilder, outputValue); } // iterate over keys that only exists in rightMap + ValueBlock rightKeyBlock = rightRawKeyBlock.getUnderlyingValueBlock(); for (int rightIndex = 0; rightIndex < rightSize; rightIndex++) { if (!keyFound[rightIndex]) { Object key = readNativeValue(keyType, rightRawKeyBlock, rightRawOffset + rightIndex); @@ -136,7 +139,7 @@ public static SqlMap mapZipWith( Object outputValue = function.apply(key, null, rightValue); - keyType.appendTo(rightRawKeyBlock, rightRawOffset + rightIndex, keyBuilder); + keyBuilder.append(rightKeyBlock, rightRawKeyBlock.getUnderlyingValuePosition(rightRawOffset + rightIndex)); writeNativeValue(outputValueType, valueBuilder, outputValue); } } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MathFunctions.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MathFunctions.java index 70dfdd7daea2..f884469c68da 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MathFunctions.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MathFunctions.java @@ -61,7 +61,6 @@ import static io.trino.spi.type.Int128Math.rescale; import static io.trino.spi.type.Int128Math.rescaleTruncate; import static io.trino.spi.type.Int128Math.subtract; -import static io.trino.spi.type.VarcharType.VARCHAR; import static io.trino.type.DecimalOperators.modulusScalarFunction; import static io.trino.util.Failures.checkCondition; import static java.lang.Character.MAX_RADIX; @@ -1441,7 +1440,7 @@ private static double mapDotProduct(BlockPositionIsIdentical varcharIdentical, B Block rightRawKeyBlock = rightMap.getRawKeyBlock(); Block rightRawValueBlock = rightMap.getRawValueBlock(); - BlockSet rightMapKeys = new BlockSet(VARCHAR, varcharIdentical, varcharHashCode, rightMap.getSize()); + BlockSet rightMapKeys = new BlockSet(varcharIdentical, varcharHashCode, rightMap.getSize()); for (int i = 0; i < rightMap.getSize(); i++) { rightMapKeys.add(rightRawKeyBlock, rightRawOffset + i); diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/MultimapFromEntriesFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/MultimapFromEntriesFunction.java index b56a10bb73fe..99ed56d2be3a 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/MultimapFromEntriesFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/MultimapFromEntriesFunction.java @@ -84,7 +84,7 @@ public SqlMap multimapFromEntries( if (entryCount > entryIndicesList.length) { initializeEntryIndicesList(entryCount); } - BlockSet keySet = new BlockSet(keyType, keysIdenticalOperator, keyHashCode, entryCount); + BlockSet keySet = new BlockSet(keysIdenticalOperator, keyHashCode, entryCount); for (int i = 0; i < entryCount; i++) { if (mapEntries.isNull(i)) { @@ -113,11 +113,13 @@ public SqlMap multimapFromEntries( IntList indexList = entryIndicesList[i]; SqlRow keyEntry = mapEntryType.getObject(mapEntries, indexList.getInt(0)); - keyType.appendTo(keyEntry.getRawFieldBlock(0), keyEntry.getRawIndex(), keyBuilder); + Block rawKeyBlock = keyEntry.getRawFieldBlock(0); + keyBuilder.append(rawKeyBlock.getUnderlyingValueBlock(), rawKeyBlock.getUnderlyingValuePosition(keyEntry.getRawIndex())); ((ArrayBlockBuilder) valueBuilder).buildEntry(elementBuilder -> { for (int entryIndex : indexList) { SqlRow valueEntry = mapEntryType.getObject(mapEntries, entryIndex); - valueType.appendTo(valueEntry.getRawFieldBlock(1), valueEntry.getRawIndex(), elementBuilder); + Block rawValueBlock = valueEntry.getRawFieldBlock(1); + elementBuilder.append(rawValueBlock.getUnderlyingValueBlock(), rawValueBlock.getUnderlyingValuePosition(valueEntry.getRawIndex())); } }); } diff --git a/core/trino-main/src/main/java/io/trino/operator/scalar/ZipFunction.java b/core/trino-main/src/main/java/io/trino/operator/scalar/ZipFunction.java index 34bbd56057ad..6f0078b98469 100644 --- a/core/trino-main/src/main/java/io/trino/operator/scalar/ZipFunction.java +++ b/core/trino-main/src/main/java/io/trino/operator/scalar/ZipFunction.java @@ -104,12 +104,12 @@ public static Block zip(List types, Block... arrays) RowType rowType = RowType.anonymous(types); RowBlockBuilder outputBuilder = rowType.createBlockBuilder(null, biggestCardinality); for (int outputPosition = 0; outputPosition < biggestCardinality; outputPosition++) { - buildRow(types, outputBuilder, outputPosition, arrays); + buildRow(outputBuilder, outputPosition, arrays); } return outputBuilder.build(); } - private static void buildRow(List types, RowBlockBuilder outputBuilder, int outputPosition, Block[] arrays) + private static void buildRow(RowBlockBuilder outputBuilder, int outputPosition, Block[] arrays) { outputBuilder.buildEntry(fieldBuilders -> { for (int fieldIndex = 0; fieldIndex < arrays.length; fieldIndex++) { @@ -117,7 +117,8 @@ private static void buildRow(List types, RowBlockBuilder outputBuilder, in fieldBuilders.get(fieldIndex).appendNull(); } else { - types.get(fieldIndex).appendTo(arrays[fieldIndex], outputPosition, fieldBuilders.get(fieldIndex)); + Block block = arrays[fieldIndex]; + fieldBuilders.get(fieldIndex).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(outputPosition)); } } }); diff --git a/core/trino-main/src/main/java/io/trino/operator/window/PatternRecognitionPartition.java b/core/trino-main/src/main/java/io/trino/operator/window/PatternRecognitionPartition.java index c06ec8666477..efe70cb9c4cd 100644 --- a/core/trino-main/src/main/java/io/trino/operator/window/PatternRecognitionPartition.java +++ b/core/trino-main/src/main/java/io/trino/operator/window/PatternRecognitionPartition.java @@ -294,7 +294,7 @@ private void outputEmptyMatch(PageBuilder pageBuilder) // compute measures for (MeasureComputation measureComputation : measures) { Block result = measureComputation.computeEmpty(matchNumber); - measureComputation.getType().appendTo(result, 0, pageBuilder.getBlockBuilder(channel)); + pageBuilder.getBlockBuilder(channel).append(result.getUnderlyingValueBlock(), result.getUnderlyingValuePosition(0)); channel++; } // window functions have empty frame @@ -323,7 +323,7 @@ private void outputOneRowPerMatch(PageBuilder pageBuilder, MatchResult matchResu ArrayView labels = matchResult.getLabels(); for (MeasureComputation measureComputation : measures) { Block result = measureComputation.compute(patternStart + labels.length() - 1, labels, partitionStart, searchStart, searchEnd, patternStart, matchNumber, measureComputationsIndex); - measureComputation.getType().appendTo(result, 0, pageBuilder.getBlockBuilder(channel)); + pageBuilder.getBlockBuilder(channel).append(result.getUnderlyingValueBlock(), result.getUnderlyingValuePosition(0)); channel++; } // window functions have frame consisting of all rows of the match @@ -374,7 +374,7 @@ private void outputRow(PageBuilder pageBuilder, ArrayView labels, int position, // compute measures from the current position (the position from which measures are computed matters in RUNNING semantics) for (MeasureComputation measureComputation : measures) { Block result = measureComputation.compute(position, labels, partitionStart, searchStart, searchEnd, currentPosition, matchNumber, measureComputationsIndex); - measureComputation.getType().appendTo(result, 0, pageBuilder.getBlockBuilder(channel)); + pageBuilder.getBlockBuilder(channel).append(result.getUnderlyingValueBlock(), result.getUnderlyingValuePosition(0)); channel++; } } diff --git a/core/trino-main/src/main/java/io/trino/operator/window/pattern/LogicalIndexNavigation.java b/core/trino-main/src/main/java/io/trino/operator/window/pattern/LogicalIndexNavigation.java index e72e48304d47..e37260319986 100644 --- a/core/trino-main/src/main/java/io/trino/operator/window/pattern/LogicalIndexNavigation.java +++ b/core/trino-main/src/main/java/io/trino/operator/window/pattern/LogicalIndexNavigation.java @@ -76,26 +76,20 @@ public int resolvePosition(int currentRow, ArrayView matchedLabels, int searchSt checkArgument(currentRow >= patternStart && currentRow < patternStart + matchedLabels.length(), "current row is out of bounds of the match"); int relativePosition; + int patternEndInclusive = running ? currentRow - patternStart : matchedLabels.length() - 1; if (last) { - int start; - if (running) { - start = currentRow - patternStart; - } - else { - start = matchedLabels.length() - 1; - } - relativePosition = findLastAndBackwards(start, matchedLabels); + relativePosition = findLastAndBackwards(patternEndInclusive, matchedLabels); } else { - relativePosition = findFirstAndForward(matchedLabels); + relativePosition = findFirstAndForward(patternEndInclusive, matchedLabels); } return adjustPosition(relativePosition, patternStart, searchStart, searchEnd); } // LAST(A.price, 3): find the last occurrence of label "A" and go 3 occurrences backwards - private int findLastAndBackwards(int searchStart, ArrayView matchedLabels) + private int findLastAndBackwards(int patternEndInclusive, ArrayView matchedLabels) { - int position = searchStart + 1; + int position = patternEndInclusive + 1; int found = 0; while (found <= logicalOffset && position > 0) { position--; @@ -110,11 +104,11 @@ private int findLastAndBackwards(int searchStart, ArrayView matchedLabels) } // FIRST(A.price, 3): find the first occurrence of label "A" and go 3 occurrences forward - private int findFirstAndForward(ArrayView matchedLabels) + private int findFirstAndForward(int patternEndInclusive, ArrayView matchedLabels) { int position = -1; int found = 0; - while (found <= logicalOffset && position < matchedLabels.length() - 1) { + while (found <= logicalOffset && position < patternEndInclusive) { position++; if (labels.isEmpty() || labels.contains(matchedLabels.get(position))) { found++; diff --git a/core/trino-main/src/main/java/io/trino/operator/window/pattern/MeasureComputation.java b/core/trino-main/src/main/java/io/trino/operator/window/pattern/MeasureComputation.java index 959748c3d9df..e469532c4a0e 100644 --- a/core/trino-main/src/main/java/io/trino/operator/window/pattern/MeasureComputation.java +++ b/core/trino-main/src/main/java/io/trino/operator/window/pattern/MeasureComputation.java @@ -50,30 +50,21 @@ public class MeasureComputation // are not used) private final Block[] nulls; - // result type - private final Type type; - // mapping from int representation to label name private final List labelNames; private final ConnectorSession session; - public MeasureComputation(PageProjection projection, List expectedLayout, List aggregations, Type type, List labelNames, ConnectorSession session) + public MeasureComputation(PageProjection projection, List expectedLayout, List aggregations, List labelNames, ConnectorSession session) { this.projection = requireNonNull(projection, "projection is null"); this.expectedLayout = requireNonNull(expectedLayout, "expectedLayout is null"); this.aggregations = aggregations.toArray(new MatchAggregation[] {}); this.nulls = precomputeNulls(expectedLayout); - this.type = requireNonNull(type, "type is null"); this.labelNames = requireNonNull(labelNames, "labelNames is null"); this.session = requireNonNull(session, "session is null"); } - public Type getType() - { - return type; - } - public Block compute(int currentRow, ArrayView matchedLabels, int partitionStart, int searchStart, int searchEnd, int patternStart, long matchNumber, ProjectingPagesWindowIndex windowIndex) { return compute(currentRow, matchedLabels, aggregations, partitionStart, searchStart, searchEnd, patternStart, matchNumber, windowIndex, projection, expectedLayout, nulls, labelNames, session); @@ -190,22 +181,20 @@ public static class MeasureComputationSupplier { private final Supplier projection; private final List expectedLayout; - private final Type type; private final List labelNames; private final ConnectorSession session; - public MeasureComputationSupplier(Supplier projection, List expectedLayout, Type type, List labelNames, ConnectorSession session) + public MeasureComputationSupplier(Supplier projection, List expectedLayout, List labelNames, ConnectorSession session) { this.projection = requireNonNull(projection, "projection is null"); this.expectedLayout = requireNonNull(expectedLayout, "expectedLayout is null"); - this.type = requireNonNull(type, "type is null"); this.labelNames = requireNonNull(labelNames, "labelNames is null"); this.session = requireNonNull(session, "session is null"); } public MeasureComputation get(List aggregations) { - return new MeasureComputation(projection.get(), expectedLayout, aggregations, type, labelNames, session); + return new MeasureComputation(projection.get(), expectedLayout, aggregations, labelNames, session); } } } diff --git a/core/trino-main/src/main/java/io/trino/operator/window/pattern/ProjectingPagesWindowIndex.java b/core/trino-main/src/main/java/io/trino/operator/window/pattern/ProjectingPagesWindowIndex.java index e71f70a2415d..c4c6b4a94af0 100644 --- a/core/trino-main/src/main/java/io/trino/operator/window/pattern/ProjectingPagesWindowIndex.java +++ b/core/trino-main/src/main/java/io/trino/operator/window/pattern/ProjectingPagesWindowIndex.java @@ -223,7 +223,8 @@ public void appendTo(int channel, int position, BlockBuilder output) pagesIndex.appendTo(channel, position(position), output); } int channelIndex = channel - firstProjectedChannel; - projectedTypes.get(channelIndex).appendTo(compute(position, channelIndex), 0, output); + Block block = compute(position, channelIndex); + output.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(0)); } @Override diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index adee6e2e7f2b..7d38137806d0 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -242,6 +242,8 @@ protected void setup(Binder binder) newExporter(binder).export(ClusterMemoryManager.class).withGeneratedName(); + jaxrsBinder(binder).bind(GatewayResource.class); + // node partitioning manager binder.bind(NodePartitioningManager.class).in(Scopes.SINGLETON); diff --git a/core/trino-main/src/main/java/io/trino/server/DataSizeSerializer.java b/core/trino-main/src/main/java/io/trino/server/DataSizeSerializer.java new file mode 100644 index 000000000000..a71ea56f1e5b --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/server/DataSizeSerializer.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.server; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import io.airlift.units.DataSize; + +import java.io.IOException; + +public class DataSizeSerializer + extends JsonSerializer +{ + public static final String SUCCINCT_DATA_SIZE_ENABLED = "dataSize.succinct.enabled"; + + @Override + public void serialize(DataSize dataSize, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException + { + if (dataSize == null) { + jsonGenerator.writeNull(); + return; + } + + if (Boolean.TRUE.equals(serializerProvider.getAttribute(SUCCINCT_DATA_SIZE_ENABLED))) { + jsonGenerator.writeString(dataSize.succinct().toString()); + return; + } + jsonGenerator.writeString(dataSize.toBytesValueString()); + } +} diff --git a/core/trino-main/src/main/java/io/trino/server/DynamicFilterService.java b/core/trino-main/src/main/java/io/trino/server/DynamicFilterService.java index 82936ea9aaee..7936910e2976 100644 --- a/core/trino-main/src/main/java/io/trino/server/DynamicFilterService.java +++ b/core/trino-main/src/main/java/io/trino/server/DynamicFilterService.java @@ -380,8 +380,8 @@ public void registerDynamicFilterConsumer(QueryId queryId, int attemptId, Set newDynamicFilters) { - DynamicFilterContext context = dynamicFilterContexts.get(taskId.getQueryId()); - int taskAttemptId = taskId.getAttemptId(); + DynamicFilterContext context = dynamicFilterContexts.get(taskId.queryId()); + int taskAttemptId = taskId.attemptId(); if (context == null || taskAttemptId < context.getAttemptId()) { // query has been removed or dynamic filters are from a previous query attempt return; @@ -389,14 +389,14 @@ public void addTaskDynamicFilters(TaskId taskId, Map ne checkState( context.isTaskRetriesEnabled() || taskAttemptId == context.getAttemptId(), "Query %s retry attempt %s has not been registered with dynamic filter service", - taskId.getQueryId(), + taskId.queryId(), taskAttemptId); context.addTaskDynamicFilters(taskId, newDynamicFilters); } public void stageCannotScheduleMoreTasks(StageId stageId, int attemptId, int numberOfTasks) { - DynamicFilterContext context = dynamicFilterContexts.get(stageId.getQueryId()); + DynamicFilterContext context = dynamicFilterContexts.get(stageId.queryId()); if (context == null || attemptId < context.getAttemptId()) { // query has been removed or not registered (e.g. dynamic filtering is disabled) // or a newer attempt has already been triggered @@ -732,7 +732,7 @@ private void collectReplicated(Domain domain) private void collectPartitioned(TaskId taskId, Domain domain) { synchronized (collectedTasks) { - if (!collectedTasks.checkedAdd(taskId.getPartitionId())) { + if (!collectedTasks.checkedAdd(taskId.partitionId())) { return; } } @@ -969,7 +969,7 @@ private void addTaskDynamicFilters(TaskId taskId, Map n collectionContext.collect(taskId, domain); }); - if (stageDynamicFilters.computeIfAbsent(taskId.getStageId(), key -> newConcurrentHashSet()).addAll(newDynamicFilters.keySet())) { + if (stageDynamicFilters.computeIfAbsent(taskId.stageId(), key -> newConcurrentHashSet()).addAll(newDynamicFilters.keySet())) { updateExpectedTaskCount(); } } diff --git a/core/trino-main/src/main/java/io/trino/server/ExternalUriInfo.java b/core/trino-main/src/main/java/io/trino/server/ExternalUriInfo.java index b1b60c92b614..01a4ed6384fd 100644 --- a/core/trino-main/src/main/java/io/trino/server/ExternalUriInfo.java +++ b/core/trino-main/src/main/java/io/trino/server/ExternalUriInfo.java @@ -13,9 +13,9 @@ */ package io.trino.server; -import jakarta.ws.rs.HeaderParam; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.UriBuilder; import jakarta.ws.rs.core.UriBuilderException; import jakarta.ws.rs.core.UriInfo; @@ -23,8 +23,8 @@ import java.net.URI; import java.net.URISyntaxException; +import static com.google.common.base.MoreObjects.firstNonNull; import static java.util.Objects.requireNonNull; -import static java.util.Objects.requireNonNullElse; /** * Provides external URI information for the current request. The external URI may have a path prefix when behind a reverse proxy. @@ -37,10 +37,15 @@ public class ExternalUriInfo private final UriInfo uriInfo; private final String forwardedPrefix; - public ExternalUriInfo(@Context UriInfo uriInfo, @HeaderParam(X_FORWARDED_PREFIX) String forwardedPrefix) + public ExternalUriInfo(@Context UriInfo uriInfo, @Context HttpHeaders httpHeaders) + { + this(uriInfo, requireNonNull(httpHeaders, "httpHeaders is null").getHeaderString(X_FORWARDED_PREFIX)); + } + + ExternalUriInfo(UriInfo uriInfo, String forwardedPrefix) { this.uriInfo = requireNonNull(uriInfo, "uriInfo is null"); - this.forwardedPrefix = requireNonNullElse(forwardedPrefix, ""); + this.forwardedPrefix = firstNonNull(forwardedPrefix, ""); } public static ExternalUriInfo from(ContainerRequestContext requestContext) diff --git a/core/trino-main/src/main/java/io/trino/server/GatewayResource.java b/core/trino-main/src/main/java/io/trino/server/GatewayResource.java new file mode 100644 index 000000000000..dc40078558fe --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/server/GatewayResource.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.server; + +import com.google.inject.Inject; +import io.trino.memory.ClusterMemoryManager; +import io.trino.memory.MemoryInfo; +import io.trino.server.security.ResourceSecurity; +import io.trino.spi.memory.MemoryPoolInfo; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; + +import java.util.Map; +import java.util.Optional; + +import static io.trino.server.security.ResourceSecurity.AccessType.MANAGEMENT_READ; +import static java.util.Objects.requireNonNull; + +@Path("/v1/integrations/gateway") +@ResourceSecurity(MANAGEMENT_READ) +public class GatewayResource +{ + private final ClusterMemoryManager clusterMemoryManager; + + @Inject + public GatewayResource(ClusterMemoryManager clusterMemoryManager) + { + this.clusterMemoryManager = requireNonNull(clusterMemoryManager, "clusterMemoryManager is null"); + } + + @GET + @Path("metrics") + public ClusterMetrics getClusterMetrics() + { + Map> memoryInfo = clusterMemoryManager.getAllNodesMemoryInfo(); + long totalFreeBytes = memoryInfo + .values() + .stream() + .flatMap(Optional::stream) + .map(MemoryInfo::getPool) + .mapToLong(MemoryPoolInfo::getFreeBytes) + .sum(); + double aggregatedSystemLoad = memoryInfo + .values() + .stream() + .flatMap(Optional::stream) + .mapToDouble(MemoryInfo::getSystemCpuLoad) + .sum(); + return new ClusterMetrics(memoryInfo.size(), totalFreeBytes, aggregatedSystemLoad); + } + + /** + * Represents metrics aggregated from all nodes in the Trino cluster + */ + public record ClusterMetrics(long clusterSize, long totalFreeBytes, double aggregatedSystemLoad) {} +} diff --git a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java index 76b27c230de3..1fa86e21fe93 100644 --- a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java +++ b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java @@ -162,6 +162,7 @@ import static com.google.inject.multibindings.Multibinder.newSetBinder; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static io.airlift.concurrent.Threads.virtualThreadsNamed; import static io.airlift.configuration.ConfigBinder.configBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; import static io.airlift.json.JsonBinder.jsonBinder; @@ -173,6 +174,7 @@ import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static java.util.concurrent.Executors.newScheduledThreadPool; +import static java.util.concurrent.Executors.newThreadPerTaskExecutor; import static java.util.concurrent.TimeUnit.SECONDS; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -409,6 +411,9 @@ protected void setup(Binder binder) jsonBinder(binder).addSerializerBinding(Slice.class).to(SliceSerializer.class); jsonBinder(binder).addDeserializerBinding(Slice.class).to(SliceDeserializer.class); + // configurable DataSize serialization + jsonBinder(binder).addSerializerBinding(DataSize.class).to(DataSizeSerializer.class); + // node version binder.bind(NodeVersion.class).toInstance(new NodeVersion(nodeVersion)); @@ -540,9 +545,7 @@ public static Executor createStartupExecutor(ServerConfig config) if (!config.isConcurrentStartup()) { return directExecutor(); } - return new BoundedExecutor( - newCachedThreadPool(daemonThreadsNamed("startup-%s")), - Runtime.getRuntime().availableProcessors()); + return newThreadPerTaskExecutor(virtualThreadsNamed("startup#v%s")); } @Provides diff --git a/core/trino-main/src/main/java/io/trino/server/ServerPluginsProvider.java b/core/trino-main/src/main/java/io/trino/server/ServerPluginsProvider.java index c1ef3d897f36..70ac9143f544 100644 --- a/core/trino-main/src/main/java/io/trino/server/ServerPluginsProvider.java +++ b/core/trino-main/src/main/java/io/trino/server/ServerPluginsProvider.java @@ -16,12 +16,12 @@ import com.google.inject.Inject; import io.trino.server.PluginManager.PluginsProvider; -import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.net.MalformedURLException; import java.net.URL; import java.nio.file.DirectoryStream; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.concurrent.Callable; @@ -36,13 +36,13 @@ public class ServerPluginsProvider implements PluginsProvider { - private final File installedPluginsDir; + private final List installedPluginsDirs; private final Executor executor; @Inject public ServerPluginsProvider(ServerPluginsProviderConfig config, @ForStartup Executor executor) { - this.installedPluginsDir = config.getInstalledPluginsDir(); + this.installedPluginsDirs = config.getInstalledPluginsDirs(); this.executor = requireNonNull(executor, "executor is null"); } @@ -51,29 +51,30 @@ public void loadPlugins(Loader loader, ClassLoaderFactory createClassLoader) { executeUntilFailure( executor, - listFiles(installedPluginsDir).stream() - .filter(File::isDirectory) - .map(file -> (Callable) () -> { - loader.load(file.getAbsolutePath(), () -> - createClassLoader.create(file.getName(), buildClassPath(file))); + installedPluginsDirs.stream() + .flatMap(installedPluginsDir -> listFiles(installedPluginsDir).stream()) + .filter(Files::isDirectory) + .map(Path::toAbsolutePath) + .map(path -> (Callable) () -> { + String name = path.getFileName().toString(); + loader.load(name, () -> createClassLoader.create(name, buildClassPath(path))); return null; }) .collect(toImmutableList())); } - private static List buildClassPath(File path) + private static List buildClassPath(Path path) { return listFiles(path).stream() .map(ServerPluginsProvider::fileToUrl) .collect(toImmutableList()); } - private static List listFiles(File path) + private static List listFiles(Path path) { try { - try (DirectoryStream directoryStream = newDirectoryStream(path.toPath())) { + try (DirectoryStream directoryStream = newDirectoryStream(path)) { return stream(directoryStream) - .map(Path::toFile) .sorted() .collect(toImmutableList()); } @@ -83,10 +84,10 @@ private static List listFiles(File path) } } - private static URL fileToUrl(File file) + private static URL fileToUrl(Path file) { try { - return file.toURI().toURL(); + return file.toUri().toURL(); } catch (MalformedURLException e) { throw new UncheckedIOException(e); diff --git a/core/trino-main/src/main/java/io/trino/server/ServerPluginsProviderConfig.java b/core/trino-main/src/main/java/io/trino/server/ServerPluginsProviderConfig.java index 5d72970e5787..4e017ac417df 100644 --- a/core/trino-main/src/main/java/io/trino/server/ServerPluginsProviderConfig.java +++ b/core/trino-main/src/main/java/io/trino/server/ServerPluginsProviderConfig.java @@ -13,23 +13,28 @@ */ package io.trino.server; +import com.google.common.collect.ImmutableList; import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.configuration.validation.FileExists; -import java.io.File; +import java.nio.file.Path; +import java.util.List; public class ServerPluginsProviderConfig { - private File installedPluginsDir = new File("plugin"); + private List installedPluginsDirs = ImmutableList.of(Path.of("plugin")); - public File getInstalledPluginsDir() + public List<@FileExists Path> getInstalledPluginsDirs() { - return installedPluginsDir; + return installedPluginsDirs; } @Config("plugin.dir") - public ServerPluginsProviderConfig setInstalledPluginsDir(File installedPluginsDir) + @ConfigDescription("Comma separated list of root directories where the plugins are located") + public ServerPluginsProviderConfig setInstalledPluginsDirs(List installedPluginsDirs) { - this.installedPluginsDir = installedPluginsDir; + this.installedPluginsDirs = ImmutableList.copyOf(installedPluginsDirs); return this; } } diff --git a/core/trino-main/src/main/java/io/trino/server/TaskResource.java b/core/trino-main/src/main/java/io/trino/server/TaskResource.java index c67b744b87d0..278dcec315b2 100644 --- a/core/trino-main/src/main/java/io/trino/server/TaskResource.java +++ b/core/trino-main/src/main/java/io/trino/server/TaskResource.java @@ -433,9 +433,9 @@ private boolean injectFailure( Optional injectedFailure = failureInjector.getInjectedFailure( traceToken.get(), - taskId.getStageId().getId(), - taskId.getPartitionId(), - taskId.getAttemptId()); + taskId.stageId().id(), + taskId.partitionId(), + taskId.attemptId()); if (injectedFailure.isEmpty()) { return false; diff --git a/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java b/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java index e8359258f690..341efa910784 100644 --- a/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java +++ b/core/trino-main/src/main/java/io/trino/server/protocol/ProtocolUtil.java @@ -219,7 +219,7 @@ private static StageStats toStageStats(StageId stageId, BasicStagesInfo stages, // Store current stage details into a builder StageStats.Builder builder = StageStats.builder() - .setStageId(String.valueOf(stageInfo.getStageId().getId())) + .setStageId(String.valueOf(stageInfo.getStageId().id())) .setState(stageInfo.getState().toString()) .setDone(stageInfo.getState().isDone()) .setTotalSplits(stageStats.getTotalDrivers()) diff --git a/core/trino-main/src/main/java/io/trino/server/protocol/Query.java b/core/trino-main/src/main/java/io/trino/server/protocol/Query.java index ab1ad24c7dac..737930f76991 100644 --- a/core/trino-main/src/main/java/io/trino/server/protocol/Query.java +++ b/core/trino-main/src/main/java/io/trino/server/protocol/Query.java @@ -523,7 +523,7 @@ private synchronized QueryResultsResponse getNextResult(long token, ExternalUriI // first time through, self is null QueryResults queryResults = new QueryResults( - queryId.toString(), + queryId.id(), getQueryInfoUri(queryInfoUrl, queryId, externalUriInfo), partialCancelUri, nextResultsUri, @@ -714,7 +714,7 @@ private URI createNextResultsUri(ExternalUriInfo externalUriInfo, long nextToken { return externalUriInfo.baseUriBuilder() .path("/v1/statement/executing") - .path(queryId.toString()) + .path(queryId.id()) .path(slug.makeSlug(EXECUTING_QUERY, nextToken)) .path(String.valueOf(nextToken)) .build(); @@ -724,7 +724,7 @@ private URI createPartialCancelUri(int stage, ExternalUriInfo externalUriInfo, l { return externalUriInfo.baseUriBuilder() .path("/v1/statement/executing/partialCancel") - .path(queryId.toString()) + .path(queryId.id()) .path(String.valueOf(stage)) .path(slug.makeSlug(EXECUTING_QUERY, nextToken)) .path(String.valueOf(nextToken)) @@ -760,7 +760,7 @@ private static Optional findCancelableLeafStage(StageId stageId, BasicS } // no matching sub stage, so return this stage - return Optional.of(stage.getStageId().getId()); + return Optional.of(stage.getStageId().id()); } private static QueryError toQueryError(ResultQueryInfo queryInfo, Optional exception) diff --git a/core/trino-main/src/main/java/io/trino/server/protocol/QueryInfoUrlFactory.java b/core/trino-main/src/main/java/io/trino/server/protocol/QueryInfoUrlFactory.java index 2bfc9fed8fbb..0b67c96a85f8 100644 --- a/core/trino-main/src/main/java/io/trino/server/protocol/QueryInfoUrlFactory.java +++ b/core/trino-main/src/main/java/io/trino/server/protocol/QueryInfoUrlFactory.java @@ -45,7 +45,7 @@ public QueryInfoUrlFactory(ServerConfig serverConfig) public Optional getQueryInfoUrl(QueryId queryId) { return queryInfoUrlTemplate - .map(template -> template.replace("${QUERY_ID}", queryId.toString())) + .map(template -> template.replace("${QUERY_ID}", queryId.id())) .map(URI::create); } @@ -54,7 +54,7 @@ public static URI getQueryInfoUri(Optional queryInfoUrl, QueryId queryId, E return queryInfoUrl.orElseGet(() -> externalUriInfo.baseUriBuilder() .path("ui/query.html") - .replaceQuery(queryId.toString()) + .replaceQuery(queryId.id()) .build()); } } diff --git a/core/trino-main/src/main/java/io/trino/server/remotetask/HttpLocationFactory.java b/core/trino-main/src/main/java/io/trino/server/remotetask/HttpLocationFactory.java index 7d5857be1b9d..28746d737ff5 100644 --- a/core/trino-main/src/main/java/io/trino/server/remotetask/HttpLocationFactory.java +++ b/core/trino-main/src/main/java/io/trino/server/remotetask/HttpLocationFactory.java @@ -50,7 +50,7 @@ public URI createQueryLocation(QueryId queryId) requireNonNull(queryId, "queryId is null"); return uriBuilderFrom(baseUri) .appendPath("/v1/query") - .appendPath(queryId.toString()) + .appendPath(queryId.id()) .build(); } diff --git a/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java b/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java index eec10df3901a..2691c3a2f4e3 100644 --- a/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java +++ b/core/trino-main/src/main/java/io/trino/server/remotetask/HttpRemoteTask.java @@ -384,8 +384,8 @@ public HttpRemoteTask( this.outboundDynamicFiltersCollector = new DynamicFiltersCollector(this::triggerUpdate); dynamicFilterService.registerDynamicFilterConsumer( - taskId.getQueryId(), - taskId.getAttemptId(), + taskId.queryId(), + taskId.attemptId(), outboundDynamicFilterIds, outboundDynamicFiltersCollector::updateDomains); @@ -1176,8 +1176,8 @@ private SpanBuilder createSpanBuilder(String name, Span parent) { return tracer.spanBuilder(name) .setParent(Context.current().with(parent)) - .setAttribute(TrinoAttributes.QUERY_ID, taskId.getQueryId().toString()) - .setAttribute(TrinoAttributes.STAGE_ID, taskId.getStageId().toString()) + .setAttribute(TrinoAttributes.QUERY_ID, taskId.queryId().toString()) + .setAttribute(TrinoAttributes.STAGE_ID, taskId.stageId().toString()) .setAttribute(TrinoAttributes.TASK_ID, taskId.toString()); } diff --git a/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java b/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java index 6e4812dc0a1c..8e5d603f5874 100644 --- a/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java +++ b/core/trino-main/src/main/java/io/trino/server/testing/TestingTrinoServer.java @@ -275,7 +275,10 @@ private TestingTrinoServer( .put("exchange.client-threads", "4") // Reduce memory footprint in tests .put("exchange.max-buffer-size", "4MB") - .put("internal-communication.shared-secret", "internal-shared-secret"); + .put("internal-communication.shared-secret", "internal-shared-secret") + .put("plugin.dir", baseDataDir + .orElseGet(TestingTrinoServer::tempDirectory) + .toString()); if (coordinator) { if (catalogMangerKind == CatalogMangerKind.DYNAMIC) { diff --git a/core/trino-main/src/main/java/io/trino/server/ui/TrimmedBasicQueryInfo.java b/core/trino-main/src/main/java/io/trino/server/ui/TrimmedBasicQueryInfo.java index ad616d45842a..7c34f15411f6 100644 --- a/core/trino-main/src/main/java/io/trino/server/ui/TrimmedBasicQueryInfo.java +++ b/core/trino-main/src/main/java/io/trino/server/ui/TrimmedBasicQueryInfo.java @@ -55,6 +55,7 @@ public class TrimmedBasicQueryInfo private final Optional queryType; private final RetryPolicy retryPolicy; private final Optional> clientTags; + private final Optional traceToken; public TrimmedBasicQueryInfo(BasicQueryInfo queryInfo) { @@ -82,6 +83,7 @@ public TrimmedBasicQueryInfo(BasicQueryInfo queryInfo) this.queryType = requireNonNull(queryInfo.getQueryType(), "queryType is null"); this.retryPolicy = requireNonNull(queryInfo.getRetryPolicy(), "retryPolicy is null"); this.clientTags = Optional.ofNullable(queryInfo.getSession().getClientTags()); + this.traceToken = requireNonNull(queryInfo.getSession().getTraceToken(), "traceToken is null"); } @JsonProperty @@ -192,6 +194,12 @@ public Optional> getClientTags() return clientTags; } + @JsonProperty + public Optional getTraceToken() + { + return traceToken; + } + @Override public String toString() { diff --git a/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java b/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java index c600efb4dc1f..35c13102b2ef 100644 --- a/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ui/UiQueryResource.java @@ -13,11 +13,18 @@ */ package io.trino.server.ui; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.cfg.ContextAttributes; import com.google.common.collect.ImmutableList; import com.google.inject.Inject; +import io.airlift.json.JsonCodec; +import io.airlift.json.JsonCodecFactory; import io.trino.dispatcher.DispatchManager; import io.trino.execution.QueryInfo; import io.trino.execution.QueryState; +import io.trino.operator.OperatorInfo; +import io.trino.plugin.base.metrics.TDigestHistogram; import io.trino.security.AccessControl; import io.trino.server.BasicQueryInfo; import io.trino.server.DisableHttpCache; @@ -26,6 +33,7 @@ import io.trino.server.security.ResourceSecurity; import io.trino.spi.QueryId; import io.trino.spi.TrinoException; +import io.trino.spi.metrics.Metric; import io.trino.spi.security.AccessDeniedException; import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.ForbiddenException; @@ -44,12 +52,16 @@ import java.util.NoSuchElementException; import java.util.Optional; +import static com.fasterxml.jackson.annotation.JsonIgnoreProperties.Value.forIgnoredProperties; import static io.trino.connector.system.KillQueryProcedure.createKillQueryException; import static io.trino.connector.system.KillQueryProcedure.createPreemptQueryException; +import static io.trino.plugin.base.metrics.TDigestHistogram.DIGEST_PROPERTY; import static io.trino.security.AccessControlUtil.checkCanKillQueryOwnedBy; import static io.trino.security.AccessControlUtil.checkCanViewQueryOwnedBy; import static io.trino.security.AccessControlUtil.filterQueries; +import static io.trino.server.DataSizeSerializer.SUCCINCT_DATA_SIZE_ENABLED; import static io.trino.server.security.ResourceSecurity.AccessType.WEB_UI; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; import static java.util.Objects.requireNonNull; @Path("/ui/api/query") @@ -57,13 +69,17 @@ @DisableHttpCache public class UiQueryResource { + private final JsonCodec queryInfoCodec; + private final JsonCodec prettyQueryInfoCodec; private final DispatchManager dispatchManager; private final AccessControl accessControl; private final HttpRequestSessionContextFactory sessionContextFactory; @Inject - public UiQueryResource(DispatchManager dispatchManager, AccessControl accessControl, HttpRequestSessionContextFactory sessionContextFactory) + public UiQueryResource(ObjectMapper objectMapper, DispatchManager dispatchManager, AccessControl accessControl, HttpRequestSessionContextFactory sessionContextFactory) { + this.queryInfoCodec = buildQueryInfoCodec(objectMapper, false); + this.prettyQueryInfoCodec = buildQueryInfoCodec(objectMapper, true); this.dispatchManager = requireNonNull(dispatchManager, "dispatchManager is null"); this.accessControl = requireNonNull(accessControl, "accessControl is null"); this.sessionContextFactory = requireNonNull(sessionContextFactory, "sessionContextFactory is null"); @@ -96,7 +112,13 @@ public Response getQueryInfo(@PathParam("queryId") QueryId queryId, @Context Htt if (queryInfo.isPresent()) { try { checkCanViewQueryOwnedBy(sessionContextFactory.extractAuthorizedIdentity(servletRequest, httpHeaders), queryInfo.get().getSession().toIdentity(), accessControl); - return Response.ok(queryInfo.get().pruneDigests()).build(); + + String queryString = servletRequest.getQueryString(); + if (queryString != null && queryString.contains("pretty")) { + // Use pretty JSON codec that reduces noise + return Response.ok(prettyQueryInfoCodec.toJson(queryInfo.get()), APPLICATION_JSON_TYPE).build(); + } + return Response.ok(queryInfoCodec.toJson(queryInfo.get()), APPLICATION_JSON_TYPE).build(); } catch (AccessDeniedException e) { throw new ForbiddenException(); @@ -144,4 +166,36 @@ private Response failQuery(QueryId queryId, TrinoException queryException, HttpS throw new GoneException(); } } + + private JsonCodec buildQueryInfoCodec(ObjectMapper objectMapper, boolean pretty) + { + JsonCodecFactory jsonCodecFactory = new JsonCodecFactory(() -> { + // Enable succinct DataSize serialization for QueryInfo to make it more human friendly + ContextAttributes attrs = ContextAttributes.getEmpty(); + if (pretty) { + attrs = attrs.withSharedAttribute(SUCCINCT_DATA_SIZE_ENABLED, Boolean.TRUE); + } + + ObjectMapper mapper = objectMapper + .copy() + .setDefaultAttributes(attrs); + // Don't serialize TDigestHistogram.digest which isn't useful and human readable + mapper.configOverride(TDigestHistogram.class).setIgnorals(forIgnoredProperties(DIGEST_PROPERTY)); + + // Do not output @class property for metric types + mapper.addMixIn(Metric.class, DropTypeInfo.class); + // Do not output @type property for OperatorInfo + mapper.addMixIn(OperatorInfo.class, DropTypeInfo.class); + return mapper; + }); + + if (pretty) { + jsonCodecFactory = jsonCodecFactory.prettyPrint(); + } + + return jsonCodecFactory.jsonCodec(QueryInfo.class); + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) + public interface DropTypeInfo {} } diff --git a/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java b/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java index aa7bef9696de..492adcbfa6cc 100644 --- a/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java +++ b/core/trino-main/src/main/java/io/trino/server/ui/WorkerResource.java @@ -107,7 +107,7 @@ public Response getThreads( @Context HttpServletRequest servletRequest, @Context HttpHeaders httpHeaders) { - QueryId queryId = task.getQueryId(); + QueryId queryId = task.queryId(); Optional queryInfo = dispatchManager.getFullQueryInfo(queryId); if (queryInfo.isPresent()) { try { diff --git a/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpiller.java b/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpiller.java index 4a7f0e49b225..be34e56051eb 100644 --- a/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpiller.java +++ b/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpiller.java @@ -13,16 +13,13 @@ */ package io.trino.spiller; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableList; import com.google.common.io.Closer; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import io.airlift.slice.OutputStreamSliceOutput; import io.airlift.slice.Slice; -import io.airlift.slice.SliceOutput; import io.airlift.units.DataSize; import io.trino.annotation.NotThreadSafe; import io.trino.execution.buffer.PageDeserializer; @@ -42,28 +39,29 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Iterators.transform; import static com.google.common.util.concurrent.Futures.immediateFuture; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static io.trino.spiller.FileSingleStreamSpillerFactory.SPILL_FILE_PREFIX; import static io.trino.spiller.FileSingleStreamSpillerFactory.SPILL_FILE_SUFFIX; -import static java.nio.file.StandardOpenOption.APPEND; import static java.util.Objects.requireNonNull; @NotThreadSafe public class FileSingleStreamSpiller implements SingleStreamSpiller { - @VisibleForTesting - static final int BUFFER_SIZE = 4 * 1024; + private final List spillFiles; + private volatile int currentFileIndex; - private final FileHolder targetFile; private final Closer closer = Closer.create(); private final PagesSerdeFactory serdeFactory; private volatile Optional encryptionKey; @@ -84,12 +82,15 @@ public FileSingleStreamSpiller( PagesSerdeFactory serdeFactory, Optional encryptionKey, ListeningExecutorService executor, - Path spillPath, + List spillPaths, SpillerStats spillerStats, SpillContext spillContext, LocalMemoryContext memoryContext, Runnable fileSystemErrorHandler) { + requireNonNull(spillPaths, "spillPaths is null"); + checkArgument(!spillPaths.isEmpty(), "spillPaths is empty"); + this.serdeFactory = requireNonNull(serdeFactory, "serdeFactory is null"); this.encryptionKey = requireNonNull(encryptionKey, "encryptionKey is null"); this.encrypted = encryptionKey.isPresent(); @@ -107,10 +108,14 @@ public FileSingleStreamSpiller( // This means we start accounting for the memory before the spiller thread allocates it, and we release the memory reservation // before/after the spiller thread allocates that memory -- -- whether before or after depends on whether writePages() is in the // middle of execution when close() is called (note that this applies to both readPages() and writePages() methods). - this.memoryContext.setBytes(BUFFER_SIZE); + this.memoryContext.setBytes((long) SpillFile.BUFFER_SIZE * spillPaths.size()); this.fileSystemErrorHandler = requireNonNull(fileSystemErrorHandler, "filesystemErrorHandler is null"); try { - this.targetFile = closer.register(new FileHolder(Files.createTempFile(spillPath, SPILL_FILE_PREFIX, SPILL_FILE_SUFFIX))); + ImmutableList.Builder builder = ImmutableList.builderWithExpectedSize(spillPaths.size()); + for (Path path : spillPaths) { + builder.add(closer.register(new SpillFile(Files.createTempFile(path, SPILL_FILE_PREFIX, SPILL_FILE_SUFFIX)))); + } + this.spillFiles = builder.build(); } catch (IOException e) { this.fileSystemErrorHandler.run(); @@ -137,61 +142,136 @@ public long getSpilledPagesInMemorySize() public Iterator getSpilledPages() { checkNoSpillInProgress(); - return readPages(); + checkState(writable.getAndSet(false), "Repeated reads are disallowed to prevent potential resource leaks"); + + try { + Optional encryptionKey = this.encryptionKey; + checkState(encrypted == encryptionKey.isPresent(), "encryptionKey has been discarded"); + + PageDeserializer deserializer = serdeFactory.createDeserializer(encryptionKey); + this.encryptionKey = Optional.empty(); + + int fileCount = spillFiles.size(); + List> iterators = new ArrayList<>(fileCount); + for (SpillFile file : spillFiles) { + iterators.add(readFilePages(deserializer, file, closer)); + } + + return new AbstractIterator<>() + { + int fileIndex; + + @Override + protected Page computeNext() + { + Iterator iterator = iterators.get(fileIndex); + if (!iterator.hasNext()) { + checkAllIteratorsExhausted(iterators); + return endOfData(); + } + + Page page = iterator.next(); + fileIndex = (fileIndex + 1) % fileCount; + return page; + } + }; + } + catch (IOException e) { + fileSystemErrorHandler.run(); + throw new TrinoException(GENERIC_INTERNAL_ERROR, "Failed to read spilled pages", e); + } } @Override public ListenableFuture> getAllSpilledPages() { - return executor.submit(() -> ImmutableList.copyOf(getSpilledPages())); + checkNoSpillInProgress(); + checkState(writable.getAndSet(false), "Repeated reads are disallowed to prevent potential resource leaks"); + + Optional encryptionKey = this.encryptionKey; + checkState(encrypted == encryptionKey.isPresent(), "encryptionKey has been discarded"); + + this.encryptionKey = Optional.empty(); + + List>> futures = new ArrayList<>(); + for (SpillFile file : spillFiles) { + futures.add(executor.submit(() -> { + PageDeserializer deserializer = serdeFactory.createDeserializer(encryptionKey); + ImmutableList.Builder pages = ImmutableList.builder(); + try (Closer closer = Closer.create()) { + readFilePages(deserializer, file, closer).forEachRemaining(pages::add); + } + return pages.build(); + })); + } + + // Combine pages from all spill files according to the round-robin order. + return Futures.transform(Futures.allAsList(futures), pagesPerFile -> { + ImmutableList.Builder builder = ImmutableList.builderWithExpectedSize(pagesPerFile.stream().mapToInt(List::size).sum()); + int fileCount = spillFiles.size(); + + List> iterators = new ArrayList<>(fileCount); + for (List pages : pagesPerFile) { + iterators.add(pages.iterator()); + } + + int fileIndex = 0; + while (iterators.get(fileIndex).hasNext()) { + builder.add(iterators.get(fileIndex).next()); + fileIndex = (fileIndex + 1) % fileCount; + } + checkAllIteratorsExhausted(iterators); + return builder.build(); + }, executor); + } + + private static void checkAllIteratorsExhausted(List> iterators) + { + iterators.forEach(iterator -> checkState(!iterator.hasNext(), "spill file iterator not fully consumed")); } - private DataSize writePages(Iterator pageIterator) + private DataSize writePages(Iterator pages) { checkState(writable.get(), "Spilling no longer allowed. The spiller has been made non-writable on first read for subsequent reads to be consistent"); Optional encryptionKey = this.encryptionKey; checkState(encrypted == encryptionKey.isPresent(), "encryptionKey has been discarded"); PageSerializer serializer = serdeFactory.createSerializer(encryptionKey); + long spilledPagesBytes = 0; - try (SliceOutput output = new OutputStreamSliceOutput(targetFile.newOutputStream(APPEND), BUFFER_SIZE)) { - while (pageIterator.hasNext()) { - Page page = pageIterator.next(); + int fileIndex = currentFileIndex; + int fileCount = spillFiles.size(); + + try { + while (pages.hasNext()) { + Page page = pages.next(); long pageSizeInBytes = page.getSizeInBytes(); + Slice serialized = serializer.serialize(page); + long serializedPageSize = serialized.length(); + + spillFiles.get(fileIndex).writeBytes(serialized); + spilledPagesBytes += pageSizeInBytes; + spilledPagesInMemorySize.addAndGet(pageSizeInBytes); - Slice serializedPage = serializer.serialize(page); - long pageSize = serializedPage.length(); - localSpillContext.updateBytes(pageSize); - spillerStats.addToTotalSpilledBytes(pageSize); - output.writeBytes(serializedPage); + localSpillContext.updateBytes(serializedPageSize); + spillerStats.addToTotalSpilledBytes(serializedPageSize); + + fileIndex = (fileIndex + 1) % fileCount; + } + + currentFileIndex = fileIndex; + + for (SpillFile file : spillFiles) { + file.closeOutput(); } } catch (UncheckedIOException | IOException e) { fileSystemErrorHandler.run(); throw new TrinoException(GENERIC_INTERNAL_ERROR, "Failed to spill pages", e); } - return DataSize.ofBytes(spilledPagesBytes); - } - private Iterator readPages() - { - checkState(writable.getAndSet(false), "Repeated reads are disallowed to prevent potential resource leaks"); - - try { - Optional encryptionKey = this.encryptionKey; - checkState(encrypted == encryptionKey.isPresent(), "encryptionKey has been discarded"); - PageDeserializer deserializer = serdeFactory.createDeserializer(encryptionKey); - // encryption key is safe to discard since it now belongs to the PageDeserializer and repeated reads are disallowed - this.encryptionKey = Optional.empty(); - InputStream input = closer.register(targetFile.newInputStream()); - Iterator pages = PagesSerdeUtil.readPages(deserializer, input); - return closeWhenExhausted(pages, input); - } - catch (IOException e) { - fileSystemErrorHandler.run(); - throw new TrinoException(GENERIC_INTERNAL_ERROR, "Failed to read spilled pages", e); - } + return DataSize.ofBytes(spilledPagesBytes); } @Override @@ -215,6 +295,17 @@ private void checkNoSpillInProgress() checkState(spillInProgress.isDone(), "spill in progress"); } + /** + * Returns an iterator that exposes all pages stored in the given file. + * Pages are lazily deserialized as the iterator is consumed. + */ + private Iterator readFilePages(PageDeserializer deserializer, SpillFile file, Closer closer) + throws IOException + { + InputStream input = closer.register(file.newInputStream()); + return transform(closeWhenExhausted(PagesSerdeUtil.readSerializedPages(input), input), deserializer::deserialize); + } + private static Iterator closeWhenExhausted(Iterator iterator, Closeable resource) { requireNonNull(iterator, "iterator is null"); diff --git a/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpillerFactory.java b/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpillerFactory.java index 75eb0651dea4..f0b6b4b0f2f8 100644 --- a/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpillerFactory.java +++ b/core/trino-main/src/main/java/io/trino/spiller/FileSingleStreamSpillerFactory.java @@ -22,6 +22,7 @@ import io.airlift.log.Logger; import io.trino.FeaturesConfig; import io.trino.cache.NonKeyEvictableLoadingCache; +import io.trino.execution.TaskManagerConfig; import io.trino.execution.buffer.CompressionCodec; import io.trino.execution.buffer.PagesSerdeFactory; import io.trino.memory.context.LocalMemoryContext; @@ -80,11 +81,12 @@ public class FileSingleStreamSpillerFactory private final SpillerStats spillerStats; private final double maxUsedSpaceThreshold; private final boolean spillEncryptionEnabled; + private final int spillFileCount; private int roundRobinIndex; private final NonKeyEvictableLoadingCache spillPathHealthCache; @Inject - public FileSingleStreamSpillerFactory(BlockEncodingSerde blockEncodingSerde, SpillerStats spillerStats, FeaturesConfig featuresConfig, NodeSpillConfig nodeSpillConfig) + public FileSingleStreamSpillerFactory(BlockEncodingSerde blockEncodingSerde, SpillerStats spillerStats, FeaturesConfig featuresConfig, NodeSpillConfig nodeSpillConfig, TaskManagerConfig taskManagerConfig) { this( listeningDecorator(newFixedThreadPool( @@ -93,6 +95,7 @@ public FileSingleStreamSpillerFactory(BlockEncodingSerde blockEncodingSerde, Spi requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"), spillerStats, featuresConfig.getSpillerSpillPaths(), + Math.min(featuresConfig.getSpillerThreads(), taskManagerConfig.getTaskConcurrency()), featuresConfig.getSpillMaxUsedSpaceThreshold(), nodeSpillConfig.getSpillCompressionCodec(), nodeSpillConfig.isSpillEncryptionEnabled()); @@ -104,6 +107,7 @@ public FileSingleStreamSpillerFactory( BlockEncodingSerde blockEncodingSerde, SpillerStats spillerStats, List spillPaths, + int spillFileCount, double maxUsedSpaceThreshold, CompressionCodec compressionCodec, boolean spillEncryptionEnabled) @@ -124,6 +128,7 @@ public FileSingleStreamSpillerFactory( throw new IllegalArgumentException(format("spill path %s is not accessible, it must be +rwx; adjust %s config property or filesystem permissions", path, SPILLER_SPILL_PATH)); } }); + this.spillFileCount = spillFileCount; this.maxUsedSpaceThreshold = maxUsedSpaceThreshold; this.spillEncryptionEnabled = spillEncryptionEnabled; this.roundRobinIndex = 0; @@ -165,14 +170,19 @@ private static void cleanupOldSpillFiles(Path path) } @Override - public SingleStreamSpiller create(List types, SpillContext spillContext, LocalMemoryContext memoryContext) + public SingleStreamSpiller create(List types, SpillContext spillContext, LocalMemoryContext memoryContext, boolean parallelSpill) { Optional encryptionKey = spillEncryptionEnabled ? Optional.of(createRandomAesEncryptionKey()) : Optional.empty(); + int spillFileCount = parallelSpill ? this.spillFileCount : 1; + ImmutableList.Builder paths = ImmutableList.builderWithExpectedSize(spillFileCount); + for (int i = 0; i < spillFileCount; i++) { + paths.add(getNextSpillPath()); + } return new FileSingleStreamSpiller( serdeFactory, encryptionKey, executor, - getNextSpillPath(), + paths.build(), spillerStats, spillContext, memoryContext, diff --git a/core/trino-main/src/main/java/io/trino/spiller/GenericPartitioningSpiller.java b/core/trino-main/src/main/java/io/trino/spiller/GenericPartitioningSpiller.java index 832e61a6ece1..eaf9c87b9b72 100644 --- a/core/trino-main/src/main/java/io/trino/spiller/GenericPartitioningSpiller.java +++ b/core/trino-main/src/main/java/io/trino/spiller/GenericPartitioningSpiller.java @@ -25,6 +25,7 @@ import io.trino.operator.SpillContext; import io.trino.spi.Page; import io.trino.spi.PageBuilder; +import io.trino.spi.block.Block; import io.trino.spi.type.Type; import it.unimi.dsi.fastutil.ints.IntArrayList; @@ -136,8 +137,8 @@ private synchronized IntArrayList partitionPage(Page page, IntPredicate spillPar PageBuilder pageBuilder = pageBuilders.get(partition); pageBuilder.declarePosition(); for (int channel = 0; channel < types.size(); channel++) { - Type type = types.get(channel); - type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel)); + Block block = page.getBlock(channel); + pageBuilder.getBlockBuilder(channel).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } } diff --git a/core/trino-main/src/main/java/io/trino/spiller/SingleStreamSpillerFactory.java b/core/trino-main/src/main/java/io/trino/spiller/SingleStreamSpillerFactory.java index a506744ba61f..4975397fc8cc 100644 --- a/core/trino-main/src/main/java/io/trino/spiller/SingleStreamSpillerFactory.java +++ b/core/trino-main/src/main/java/io/trino/spiller/SingleStreamSpillerFactory.java @@ -21,11 +21,16 @@ public interface SingleStreamSpillerFactory { - SingleStreamSpiller create(List types, SpillContext spillContext, LocalMemoryContext memoryContext); + default SingleStreamSpiller create(List types, SpillContext spillContext, LocalMemoryContext memoryContext) + { + return create(types, spillContext, memoryContext, false); + } + + SingleStreamSpiller create(List types, SpillContext spillContext, LocalMemoryContext memoryContext, boolean parallelSpill); static SingleStreamSpillerFactory unsupportedSingleStreamSpillerFactory() { - return (types, spillContext, memoryContext) -> { + return (types, spillContext, memoryContext, parallelSpill) -> { throw new UnsupportedOperationException(); }; } diff --git a/core/trino-main/src/main/java/io/trino/spiller/FileHolder.java b/core/trino-main/src/main/java/io/trino/spiller/SpillFile.java similarity index 62% rename from core/trino-main/src/main/java/io/trino/spiller/FileHolder.java rename to core/trino-main/src/main/java/io/trino/spiller/SpillFile.java index 06771b60e32a..1b222eb38f7c 100644 --- a/core/trino-main/src/main/java/io/trino/spiller/FileHolder.java +++ b/core/trino-main/src/main/java/io/trino/spiller/SpillFile.java @@ -13,47 +13,69 @@ */ package io.trino.spiller; +import com.google.common.annotations.VisibleForTesting; import com.google.errorprone.annotations.ThreadSafe; import com.google.errorprone.annotations.concurrent.GuardedBy; +import io.airlift.slice.OutputStreamSliceOutput; +import io.airlift.slice.Slice; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.file.Files; -import java.nio.file.OpenOption; import java.nio.file.Path; import static com.google.common.base.Preconditions.checkState; +import static java.nio.file.StandardOpenOption.APPEND; import static java.util.Objects.requireNonNull; @ThreadSafe -final class FileHolder +final class SpillFile implements Closeable { + @VisibleForTesting + static final int BUFFER_SIZE = 4 * 1024; + private final Path filePath; @GuardedBy("this") private boolean deleted; - public FileHolder(Path filePath) + @GuardedBy("this") + private OutputStreamSliceOutput output; + + public SpillFile(Path filePath) { this.filePath = requireNonNull(filePath, "filePath is null"); } - public synchronized OutputStream newOutputStream(OpenOption... options) + public synchronized void writeBytes(Slice slice) throws IOException { + requireNonNull(slice, "slice is null"); checkState(!deleted, "File already deleted"); - return Files.newOutputStream(filePath, options); + if (output == null) { + output = new OutputStreamSliceOutput(Files.newOutputStream(filePath, APPEND), BUFFER_SIZE); + } + output.writeBytes(slice); + } + + public synchronized void closeOutput() + throws IOException + { + if (output != null) { + output.close(); + output = null; + } } - public synchronized InputStream newInputStream(OpenOption... options) + public synchronized InputStream newInputStream() throws IOException { checkState(!deleted, "File already deleted"); - return Files.newInputStream(filePath, options); + closeOutput(); + return Files.newInputStream(filePath); } @Override @@ -65,6 +87,7 @@ public synchronized void close() deleted = true; try { + closeOutput(); Files.delete(filePath); } catch (IOException e) { diff --git a/core/trino-main/src/main/java/io/trino/split/PageSinkId.java b/core/trino-main/src/main/java/io/trino/split/PageSinkId.java index 69f731446e79..120f173a27fb 100644 --- a/core/trino-main/src/main/java/io/trino/split/PageSinkId.java +++ b/core/trino-main/src/main/java/io/trino/split/PageSinkId.java @@ -25,10 +25,10 @@ public class PageSinkId public static PageSinkId fromTaskId(TaskId taskId) { - long stageId = taskId.getStageId().getId(); - long partitionId = taskId.getPartitionId(); + long stageId = taskId.stageId().id(); + long partitionId = taskId.partitionId(); checkArgument(partitionId == (partitionId & 0x00FFFFFF), "partitionId is out of allowable range"); - long attemptId = taskId.getAttemptId(); + long attemptId = taskId.attemptId(); checkArgument(attemptId == (attemptId & 0xFF), "attemptId is out of allowable range"); long id = (stageId << 32) + (partitionId << 8) + attemptId; return new PageSinkId(id); diff --git a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java index e99c495475e9..4138196025c8 100644 --- a/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java +++ b/core/trino-main/src/main/java/io/trino/sql/analyzer/StatementAnalyzer.java @@ -1323,6 +1323,9 @@ protected Scope visitTableExecute(TableExecute node, Optional scope) analysis.setUpdateType("ALTER TABLE EXECUTE"); analysis.setUpdateTarget(executeHandle.catalogHandle().getVersion(), tableName, Optional.of(table), Optional.empty()); + if (!procedureMetadata.getExecutionMode().isReadsData()) { + return createAndAssignScope(node, scope, Field.newUnqualified("metric_name", VARCHAR), Field.newUnqualified("metric_value", BIGINT)); + } return createAndAssignScope(node, scope, Field.newUnqualified("rows", BIGINT)); } @@ -1656,6 +1659,7 @@ else if (expressionType instanceof MapType mapType) { outputFields.addAll(expressionOutputs); mappings.put(NodeRef.of(expression), expressionOutputs); + expressionOutputs.forEach(field -> analysis.addSourceColumns(field, analysis.getExpressionSourceColumns(expression))); } Optional ordinalityField = Optional.empty(); diff --git a/core/trino-main/src/main/java/io/trino/sql/gen/BytecodeUtils.java b/core/trino-main/src/main/java/io/trino/sql/gen/BytecodeUtils.java index 1a85c0c7025b..654250cf077e 100644 --- a/core/trino-main/src/main/java/io/trino/sql/gen/BytecodeUtils.java +++ b/core/trino-main/src/main/java/io/trino/sql/gen/BytecodeUtils.java @@ -13,6 +13,7 @@ */ package io.trino.sql.gen; +import com.google.common.base.CharMatcher; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Primitives; @@ -67,6 +68,14 @@ public final class BytecodeUtils { + private static final CharMatcher DISALLOWED_IDENTIFIER_CHARS = CharMatcher.inRange('a', 'z') + .or(CharMatcher.inRange('A', 'Z')) + .or(CharMatcher.inRange('0', '9')) + .or(CharMatcher.is('_')) + .or(CharMatcher.is('$')) + .negate() + .precomputed(); + private BytecodeUtils() {} public static BytecodeNode ifWasNullPopAndGoto(Scope scope, LabelNode label, Class returnType, Class... stackArgsToPop) @@ -452,7 +461,7 @@ public static BytecodeExpression invoke(Binding binding, BoundSignature signatur */ public static String sanitizeName(String name) { - return name.replaceAll("[^A-Za-z0-9_$]", "_"); + return DISALLOWED_IDENTIFIER_CHARS.replaceFrom(name, '_'); } public static BytecodeNode generateWrite(CallSiteBinder callSiteBinder, Scope scope, Variable wasNullVariable, Type type) diff --git a/core/trino-main/src/main/java/io/trino/sql/gen/JoinCompiler.java b/core/trino-main/src/main/java/io/trino/sql/gen/JoinCompiler.java index 90d0dc52cdf6..98ce02eddd1b 100644 --- a/core/trino-main/src/main/java/io/trino/sql/gen/JoinCompiler.java +++ b/core/trino-main/src/main/java/io/trino/sql/gen/JoinCompiler.java @@ -49,6 +49,7 @@ import io.trino.spi.PageBuilder; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.type.Type; import io.trino.spi.type.TypeOperators; import io.trino.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory; @@ -75,6 +76,7 @@ import static io.airlift.bytecode.Access.a; import static io.airlift.bytecode.Parameter.arg; import static io.airlift.bytecode.ParameterizedType.type; +import static io.airlift.bytecode.expression.BytecodeExpressions.add; import static io.airlift.bytecode.expression.BytecodeExpressions.constantClass; import static io.airlift.bytecode.expression.BytecodeExpressions.constantFalse; import static io.airlift.bytecode.expression.BytecodeExpressions.constantInt; @@ -93,7 +95,6 @@ import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL; import static io.trino.spi.function.InvocationConvention.simpleConvention; import static io.trino.sql.gen.Bootstrap.BOOTSTRAP_METHOD; -import static io.trino.sql.gen.SqlTypeBytecodeExpression.constantType; import static io.trino.util.CompilerUtils.defineClass; import static io.trino.util.CompilerUtils.makeClassName; import static java.util.Objects.requireNonNull; @@ -237,7 +238,7 @@ private Class internalCompileHashStrategy(List types, List outputChannels, List channelFields) + private static void generateAppendToMethod(ClassDefinition classDefinition, List outputChannels, List channelFields) { Parameter blockIndex = arg("blockIndex", int.class); Parameter blockPosition = arg("blockPosition", int.class); @@ -334,26 +335,23 @@ private static void generateAppendToMethod(ClassDefinition classDefinition, Call BytecodeBlock appendToBody = appendToMethod.getBody(); int pageBuilderOutputChannel = 0; + Variable block = appendToMethod.getScope().declareVariable(Block.class, "block"); for (int outputChannel : outputChannels) { - Type type = types.get(outputChannel); - BytecodeExpression typeExpression = constantType(callSiteBinder, type); - - BytecodeExpression block = thisVariable - .getField(channelFields.get(outputChannel)) - .invoke("get", Object.class, blockIndex) - .cast(Block.class); + appendToBody.append(block.set( + thisVariable.getField(channelFields.get(outputChannel)) + .invoke("get", Object.class, blockIndex) + .cast(Block.class))); + BytecodeExpression blockBuilderExpression = pageBuilder + .invoke("getBlockBuilder", BlockBuilder.class, add(outputChannelOffset, constantInt(pageBuilderOutputChannel))); appendToBody - .comment("%s.appendTo(channel_%s.get(outputChannel), blockPosition, pageBuilder.getBlockBuilder(outputChannelOffset + %s));", type.getClass(), outputChannel, pageBuilderOutputChannel) - .append(typeExpression) - .append(block) - .append(blockPosition) - .append(pageBuilder) - .append(outputChannelOffset) - .push(pageBuilderOutputChannel++) - .append(OpCode.IADD) - .invokeVirtual(PageBuilder.class, "getBlockBuilder", BlockBuilder.class, int.class) - .invokeInterface(Type.class, "appendTo", void.class, Block.class, int.class, BlockBuilder.class); + .comment("pageBuilder.getBlockBuilder(outputChannelOffset + %s).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(blockPosition));", pageBuilderOutputChannel) + .append(blockBuilderExpression.invoke( + "append", + void.class, + block.invoke("getUnderlyingValueBlock", ValueBlock.class), + block.invoke("getUnderlyingValuePosition", int.class, blockPosition))); + pageBuilderOutputChannel++; } appendToBody.ret(); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/ExpressionExtractor.java b/core/trino-main/src/main/java/io/trino/sql/planner/ExpressionExtractor.java index 04342961b649..593955a1fe86 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/ExpressionExtractor.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/ExpressionExtractor.java @@ -110,7 +110,7 @@ public Void visitFilter(FilterNode node, Void context) @Override public Void visitProject(ProjectNode node, Void context) { - node.getAssignments().getExpressions().forEach(consumer); + node.getAssignments().expressions().forEach(consumer); return super.visitProject(node, context); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java index e58ff61507ce..68e0358de5b4 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LocalExecutionPlanner.java @@ -772,7 +772,7 @@ private List getDriverFactories() public StageId getStageId() { - return taskContext.getTaskId().getStageId(); + return taskContext.getTaskId().stageId(); } public TaskId getTaskId() @@ -1494,7 +1494,7 @@ public PhysicalOperation visitPatternRecognition(PatternRecognitionNode node, Lo measureComputationsAggregationArguments.addAll(valueAccessors.getAggregationArguments()); // build measure computation - measuresBuilder.add(new MeasureComputationSupplier(pageProjectionSupplier, valueAccessors.getValueAccessors(), measure.getType(), labelNames, connectorSession)); + measuresBuilder.add(new MeasureComputationSupplier(pageProjectionSupplier, valueAccessors.getValueAccessors(), labelNames, connectorSession)); } List measureComputations = measuresBuilder.build(); @@ -3297,7 +3297,7 @@ public PhysicalOperation visitSemiJoin(SemiJoinNode node, LocalExecutionPlanCont private static Set getCoordinatorDynamicFilters(Set dynamicFilters, PlanNode node, TaskId taskId) { - if (!isBuildSideReplicated(node) || taskId.getPartitionId() == 0) { + if (!isBuildSideReplicated(node) || taskId.partitionId() == 0) { // replicated dynamic filters are collected by single stage task only return dynamicFilters; } @@ -3476,7 +3476,8 @@ public PhysicalOperation visitSimpleTableExecuteNode(SimpleTableExecuteNode node node.getId(), metadata, session, - node.getExecuteHandle()); + node.getExecuteHandle(), + getSymbolTypes(node.getOutputSymbols())); return new PhysicalOperation(operatorFactory, makeLayout(node)); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/LogicalPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/LogicalPlanner.java index 99a0718ef587..f75ac8d768ff 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/LogicalPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/LogicalPlanner.java @@ -489,7 +489,7 @@ private RelationPlan createTableCreationPlan(Analysis analysis, Query query) .map(ColumnMetadata::getName) .collect(toImmutableList()); - TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogHandle, tableMetadata); + TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogHandle, tableMetadata, create.isReplace()); return createTableWriterPlan( analysis, @@ -600,7 +600,7 @@ private RelationPlan getInsertPlan( .map(ColumnMetadata::getName) .collect(toImmutableList()); - TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, tableHandle.catalogHandle(), tableMetadata.metadata()); + TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, tableHandle.catalogHandle(), tableMetadata.metadata(), false); if (materializedViewRefreshWriterTarget.isPresent()) { RefreshType refreshType = IncrementalRefreshVisitor.canIncrementallyRefresh(plan.getRoot()); @@ -966,7 +966,9 @@ private RelationPlan createTableExecutePlan(Analysis analysis, TableExecute stat if (!analysis.isTableExecuteReadsData()) { SimpleTableExecuteNode node = new SimpleTableExecuteNode( idAllocator.getNextId(), - symbolAllocator.newSymbol("rows", BIGINT), + ImmutableList.of( + symbolAllocator.newSymbol("metricName", VARCHAR), + symbolAllocator.newSymbol("metricValue", BIGINT)), executeHandle); return new RelationPlan(node, analysis.getRootScope(), node.getOutputSymbols(), Optional.empty()); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/PlanFragmenter.java b/core/trino-main/src/main/java/io/trino/sql/planner/PlanFragmenter.java index e9c5efa78ca8..1be14868c733 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/PlanFragmenter.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/PlanFragmenter.java @@ -414,11 +414,17 @@ public PlanNode visitMergeWriter(MergeWriterNode node, RewriteContext context) { - // An empty values node is compatible with any distribution, so - // don't attempt to overwrite one's already been chosen - if (node.getRowCount() != 0 || !context.get().hasDistribution()) { + if (node.getRowCount() != 0) { + // A non-empty values node requires single distribution context.get().setSingleNodeDistribution(); } + else { + // An empty values node is compatible with any distribution, so + // do not overwrite a distribution if there is one already chosen, + // and delay setting the distribution in case the fragment contains + // another node with specific distribution requirements + context.get().setContainsEmptyValues(); + } return context.defaultRewrite(node, context.get()); } @@ -594,6 +600,7 @@ private static class FragmentProperties private final PartitioningScheme partitioningScheme; private Optional partitioningHandle = Optional.empty(); + private boolean containsEmptyValues; private Optional partitionCount = Optional.empty(); private final Set partitionedSources = new HashSet<>(); @@ -762,6 +769,11 @@ public FragmentProperties addChildren(List children) return this; } + public void setContainsEmptyValues() + { + this.containsEmptyValues = true; + } + public PartitioningScheme getPartitioningScheme() { return partitioningScheme; @@ -769,7 +781,9 @@ public PartitioningScheme getPartitioningScheme() public PartitioningHandle getPartitioningHandle() { - return partitioningHandle.get(); + checkState(partitioningHandle.isPresent() || containsEmptyValues, "PartitioningHandle is not set for a fragment that does not contain empty values"); + + return partitioningHandle.orElse(SINGLE_DISTRIBUTION); } public Optional getPartitionCount() diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/QueryPlanner.java b/core/trino-main/src/main/java/io/trino/sql/planner/QueryPlanner.java index 1c7e20ab8cdb..6a6a46bc302d 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/QueryPlanner.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/QueryPlanner.java @@ -51,7 +51,6 @@ import io.trino.sql.ir.Expression; import io.trino.sql.ir.FieldReference; import io.trino.sql.ir.IsNull; -import io.trino.sql.ir.Logical; import io.trino.sql.ir.Row; import io.trino.sql.ir.WhenClause; import io.trino.sql.planner.RelationPlanner.PatternRecognitionComponents; @@ -751,9 +750,9 @@ public MergeWriterNode plan(Merge merge) .process(merge.getTarget()); // Assign a unique id to every target table row - Symbol uniqueIdSymbol = symbolAllocator.newSymbol("unique_id", BIGINT); + Symbol targetUniqueIdSymbol = symbolAllocator.newSymbol("target_unique_id", BIGINT); RelationPlan planWithUniqueId = new RelationPlan( - new AssignUniqueId(idAllocator.getNextId(), targetTablePlan.getRoot(), uniqueIdSymbol), + new AssignUniqueId(idAllocator.getNextId(), targetTablePlan.getRoot(), targetUniqueIdSymbol), mergeAnalysis.getTargetTableScope(), targetTablePlan.getFieldMappings(), outerContext); @@ -774,8 +773,16 @@ public MergeWriterNode plan(Merge merge) RelationPlan source = new RelationPlanner(analysis, symbolAllocator, idAllocator, lambdaDeclarationToSymbolMap, plannerContext, outerContext, session, recursiveSubqueries) .process(merge.getSource()); + // Assign a unique id to every source table row + Symbol sourceUniqueIdSymbol = symbolAllocator.newSymbol("source_unique_id", BIGINT); + RelationPlan sourcePlanWithUniqueId = new RelationPlan( + new AssignUniqueId(idAllocator.getNextId(), source.getRoot(), sourceUniqueIdSymbol), + source.getScope(), + source.getFieldMappings(), + outerContext); + RelationPlan joinPlan = new RelationPlanner(analysis, symbolAllocator, idAllocator, lambdaDeclarationToSymbolMap, plannerContext, outerContext, session, recursiveSubqueries) - .planJoin(merge.getPredicate(), Join.Type.RIGHT, mergeAnalysis.getJoinScope(), planWithPresentColumn, source, analysis.getSubqueries(merge)); // TODO: ir + .planJoin(merge.getPredicate(), Join.Type.RIGHT, mergeAnalysis.getJoinScope(), planWithPresentColumn, sourcePlanWithUniqueId, analysis.getSubqueries(merge)); // TODO: ir PlanBuilder subPlan = newPlanBuilder(joinPlan, analysis, lambdaDeclarationToSymbolMap, session, plannerContext); @@ -864,10 +871,11 @@ public MergeWriterNode plan(Merge merge) List constraints = analysis.getCheckConstraints(mergeAnalysis.getTargetTable()); if (!constraints.isEmpty()) { - assignments.putIdentity(uniqueIdSymbol); + assignments.putIdentity(targetUniqueIdSymbol); + assignments.putIdentity(sourceUniqueIdSymbol); assignments.putIdentity(presentColumn); assignments.putIdentity(rowIdSymbol); - assignments.putIdentities(source.getFieldMappings()); + assignments.putIdentities(sourcePlanWithUniqueId.getFieldMappings()); subPlan = subPlan.withNewRoot(new ProjectNode( idAllocator.getNextId(), subPlan.getRoot(), @@ -897,7 +905,15 @@ public MergeWriterNode plan(Merge merge) Symbol symbol = planWithPresentColumn.getFieldMappings().get(fieldIndex); projectionAssignmentsBuilder.putIdentity(symbol); } - projectionAssignmentsBuilder.putIdentity(uniqueIdSymbol); + + // Assigns a unique id to each joined row + // The target table unique_id for matches, and the source table unique_id for non-matches + // Avoid the scenario where unique_id values become null after right join due to unmatched rows. + // It can improve performance and parallelism when handling non-matches in a mark distinct operation. + Symbol uniqueIdSymbol = symbolAllocator.newSymbol("unique_id", BIGINT); + Expression uniqueIdExpression = new Coalesce(targetUniqueIdSymbol.toSymbolReference(), sourceUniqueIdSymbol.toSymbolReference()); + + projectionAssignmentsBuilder.put(uniqueIdSymbol, uniqueIdExpression); projectionAssignmentsBuilder.putIdentity(rowIdSymbol); projectionAssignmentsBuilder.put(mergeRowSymbol, caseExpression); @@ -919,11 +935,10 @@ public MergeWriterNode plan(Merge merge) Symbol isDistinctSymbol = symbolAllocator.newSymbol("is_distinct", BOOLEAN); MarkDistinctNode markDistinctNode = new MarkDistinctNode(idAllocator.getNextId(), project, isDistinctSymbol, ImmutableList.of(uniqueIdSymbol, caseNumberSymbol)); - // Raise an error if unique_id symbol is non-null and the unique_id/case_number combination was not distinct + // The unique_id which originates from either the source or target table will not be null + // Raise an error if the unique_id/case_number combination was not distinct Expression filter = ifExpression( - Logical.and( - not(metadata, isDistinctSymbol.toSymbolReference()), - not(metadata, new IsNull(uniqueIdSymbol.toSymbolReference()))), + not(metadata, isDistinctSymbol.toSymbolReference()), new Cast( failFunction(metadata, MERGE_TARGET_ROW_MULTIPLE_MATCHES, "One MERGE target table row matched more than one source row"), BOOLEAN), @@ -1301,7 +1316,7 @@ private PlanBuilder planAggregation(PlanBuilder subPlan, List> grou } } - ImmutableList.Builder groupingKeys = ImmutableList.builder(); + ImmutableSet.Builder groupingKeys = ImmutableSet.builder(); groupingSets.stream() .flatMap(List::stream) .distinct() diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/IterativeOptimizer.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/IterativeOptimizer.java index f368bb4e0790..0158cfbb21cf 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/IterativeOptimizer.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/IterativeOptimizer.java @@ -178,11 +178,9 @@ private boolean exploreNode(int group, Context context, Set changedP invoked = true; Rule.Result result = transform(node, rule, context); timeEnd = nanoTime(); - if (result.getTransformedPlan().isPresent()) { - changedPlanNodeIds.add(result.getTransformedPlan().get().getId()); - } - if (result.getTransformedPlan().isPresent()) { - node = context.memo.replace(group, result.getTransformedPlan().get(), rule.getClass().getName()); + if (result.isPresent()) { + changedPlanNodeIds.add(result.transformedPlan().get().getId()); + node = context.memo.replace(group, result.transformedPlan().get(), rule.getClass().getName()); applied = true; done = false; @@ -226,7 +224,7 @@ private Rule.Result transform(PlanNode node, Rule rule, Context context) 0, false), PlanPrinter.textLogicalPlan( - result.getTransformedPlan().get(), + result.transformedPlan().get(), plannerContext.getMetadata(), plannerContext.getFunctionManager(), StatsAndCosts.empty(), @@ -243,7 +241,7 @@ private Rule.Result transform(PlanNode node, Rule rule, Context context) } stats.record(rule, duration, !result.isEmpty()); - if (result.getTransformedPlan().isPresent()) { + if (result.isPresent()) { return result; } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/Rule.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/Rule.java index fba94f3cfd02..f4adb1336aad 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/Rule.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/Rule.java @@ -60,7 +60,7 @@ interface Context WarningCollector getWarningCollector(); } - final class Result + record Result(Optional transformedPlan) { private static final Result EMPTY = new Result(Optional.empty()); @@ -74,21 +74,19 @@ public static Result ofPlanNode(PlanNode transformedPlan) return new Result(Optional.of(transformedPlan)); } - private final Optional transformedPlan; - - private Result(Optional transformedPlan) + public Result { - this.transformedPlan = requireNonNull(transformedPlan, "transformedPlan is null"); + requireNonNull(transformedPlan, "transformedPlan is null"); } - public Optional getTransformedPlan() + public boolean isEmpty() { - return transformedPlan; + return transformedPlan.isEmpty(); } - public boolean isEmpty() + public boolean isPresent() { - return transformedPlan.isEmpty(); + return transformedPlan.isPresent(); } } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleIndex.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleIndex.java index ce71b98c1bf8..db58d149dbaf 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleIndex.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleIndex.java @@ -21,7 +21,7 @@ import com.google.common.reflect.TypeToken; import io.trino.cache.EvictableCacheBuilder; import io.trino.matching.Pattern; -import io.trino.matching.pattern.TypeOfPattern; +import io.trino.matching.TypeOfPattern; import java.util.Set; import java.util.concurrent.ExecutionException; diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleStats.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleStats.java index 1cdd6dde402a..05ea8b4f4628 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleStats.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/RuleStats.java @@ -18,40 +18,40 @@ import org.weakref.jmx.Nested; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; public class RuleStats { - private final AtomicLong invocations = new AtomicLong(); - private final AtomicLong hits = new AtomicLong(); + private final LongAdder invocations = new LongAdder(); + private final LongAdder hits = new LongAdder(); private final TimeDistribution time = new TimeDistribution(TimeUnit.MICROSECONDS); - private final AtomicLong failures = new AtomicLong(); + private final LongAdder failures = new LongAdder(); public void record(long nanos, boolean match) { if (match) { - hits.incrementAndGet(); + hits.increment(); } - invocations.incrementAndGet(); + invocations.increment(); time.add(nanos); } public void recordFailure() { - failures.incrementAndGet(); + failures.increment(); } @Managed public long getInvocations() { - return invocations.get(); + return invocations.sum(); } @Managed public long getHits() { - return hits.get(); + return hits.sum(); } @Managed @@ -64,6 +64,6 @@ public TimeDistribution getTime() @Managed public long getFailures() { - return failures.get(); + return failures.sum(); } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateInnerUnnestWithGlobalAggregation.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateInnerUnnestWithGlobalAggregation.java index 1ad667da1f69..f7e2ad00bc77 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateInnerUnnestWithGlobalAggregation.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateInnerUnnestWithGlobalAggregation.java @@ -261,7 +261,7 @@ private static boolean isSupportedUnnest(PlanNode node, List correlation PlanNode unnestSource = lookup.resolve(unnestNode.getSource()); Set correlationSymbols = ImmutableSet.copyOf(correlation); boolean basedOnCorrelation = correlationSymbols.containsAll(unnestSymbols) || - unnestSource instanceof ProjectNode projectNode && correlationSymbols.containsAll(SymbolsExtractor.extractUnique(projectNode.getAssignments().getExpressions())); + unnestSource instanceof ProjectNode projectNode && correlationSymbols.containsAll(SymbolsExtractor.extractUnique(projectNode.getAssignments().expressions())); return isScalar(unnestNode.getSource(), lookup) && unnestNode.getReplicateSymbols().isEmpty() && diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateLeftUnnestWithGlobalAggregation.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateLeftUnnestWithGlobalAggregation.java index 14aac2c8bfa7..acd30e49deec 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateLeftUnnestWithGlobalAggregation.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateLeftUnnestWithGlobalAggregation.java @@ -217,7 +217,7 @@ private static boolean isSupportedUnnest(PlanNode node, List correlation .collect(toImmutableList()); PlanNode unnestSource = lookup.resolve(unnestNode.getSource()); boolean basedOnCorrelation = ImmutableSet.copyOf(correlation).containsAll(unnestSymbols) || - unnestSource instanceof ProjectNode projectNode && ImmutableSet.copyOf(correlation).containsAll(SymbolsExtractor.extractUnique(projectNode.getAssignments().getExpressions())); + unnestSource instanceof ProjectNode projectNode && ImmutableSet.copyOf(correlation).containsAll(SymbolsExtractor.extractUnique(projectNode.getAssignments().expressions())); return isScalar(unnestNode.getSource(), lookup) && unnestNode.getReplicateSymbols().isEmpty() && diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateUnnest.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateUnnest.java index 691cdf318829..c6be2a3a0cc9 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateUnnest.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DecorrelateUnnest.java @@ -296,7 +296,7 @@ private static boolean isSupportedUnnest(PlanNode node, List correlation .collect(toImmutableList()); PlanNode unnestSource = lookup.resolve(unnestNode.getSource()); boolean basedOnCorrelation = ImmutableSet.copyOf(correlation).containsAll(unnestSymbols) || - unnestSource instanceof ProjectNode projectNode && ImmutableSet.copyOf(correlation).containsAll(SymbolsExtractor.extractUnique(projectNode.getAssignments().getExpressions())); + unnestSource instanceof ProjectNode projectNode && ImmutableSet.copyOf(correlation).containsAll(SymbolsExtractor.extractUnique(projectNode.getAssignments().expressions())); return isScalar(unnestNode.getSource(), lookup) && unnestNode.getReplicateSymbols().isEmpty() && diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DistinctAggregationStrategyChooser.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DistinctAggregationStrategyChooser.java index 5f34790157b2..bbf01ea77362 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DistinctAggregationStrategyChooser.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/DistinctAggregationStrategyChooser.java @@ -217,9 +217,9 @@ private static boolean isAdditionalReadOverheadTooExpensive(AggregationNode aggr double subqueriesTotalDataSize = additionalColumnsDataSize * subqueryCount + distinctInputDataSize; return isNaN(subqueriesTotalDataSize) || - isNaN(singleTableScanDataSize) || - // we would read more than 50% more data - subqueriesTotalDataSize / singleTableScanDataSize > 1.5; + isNaN(singleTableScanDataSize) || + // we would read more than 50% more data + subqueriesTotalDataSize / singleTableScanDataSize > 1.5; } private static boolean isSelective(FilterNode filterNode, StatsProvider statsProvider) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ExtractDereferencesFromFilterAboveScan.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ExtractDereferencesFromFilterAboveScan.java index 079b52700401..ef31aa41b84d 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ExtractDereferencesFromFilterAboveScan.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ExtractDereferencesFromFilterAboveScan.java @@ -84,7 +84,7 @@ public Result apply(FilterNode node, Captures captures, Context context) } Assignments assignments = Assignments.of(dereferences, context.getSymbolAllocator()); - Map mappings = HashBiMap.create(assignments.getMap()) + Map mappings = HashBiMap.create(assignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherAndMergeWindows.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherAndMergeWindows.java index 686c406d1c76..524524e9c38e 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherAndMergeWindows.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherAndMergeWindows.java @@ -139,7 +139,7 @@ protected static Optional pullWindowNodeAboveProjects( // The only kind of use of the output of the target that we can safely ignore is a simple identity propagation. // The target node, when hoisted above the projections, will provide the symbols directly. Map assignmentsWithoutTargetOutputIdentities = Maps.filterKeys( - project.getAssignments().getMap(), + project.getAssignments().assignments(), output -> !(project.getAssignments().isIdentity(output) && targetOutputs.contains(output))); if (targetInputs.stream().anyMatch(assignmentsWithoutTargetOutputIdentities::containsKey)) { @@ -152,7 +152,7 @@ protected static Optional pullWindowNodeAboveProjects( .putIdentities(targetInputs) .build(); - if (!newTargetChildOutputs.containsAll(SymbolsExtractor.extractUnique(newAssignments.getExpressions()))) { + if (!newTargetChildOutputs.containsAll(SymbolsExtractor.extractUnique(newAssignments.expressions()))) { // Projection uses an output of the target -- can't move the target above this projection. return Optional.empty(); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherPartialTopN.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherPartialTopN.java index 29fad33512fc..a6c197693979 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherPartialTopN.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/GatherPartialTopN.java @@ -68,16 +68,16 @@ public class GatherPartialTopN private static boolean isGatherLocalExchange(PlanNode source) { return source instanceof ExchangeNode exchange - && exchange.getScope().equals(LOCAL) - && exchange.getType().equals(GATHER); + && exchange.getScope().equals(LOCAL) + && exchange.getType().equals(GATHER); } private static boolean isGatherRemoteExchange(ExchangeNode exchangeNode) { return exchangeNode.getScope().equals(REMOTE) - && exchangeNode.getType().equals(GATHER) - // non-empty orderingScheme means it's a merging exchange - && exchangeNode.getOrderingScheme().isEmpty(); + && exchangeNode.getType().equals(GATHER) + // non-empty orderingScheme means it's a merging exchange + && exchangeNode.getOrderingScheme().isEmpty(); } @Override diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjectIntoFilter.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjectIntoFilter.java index 7867e2fbe608..48aa18063290 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjectIntoFilter.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjectIntoFilter.java @@ -159,14 +159,14 @@ public Result apply(FilterNode node, Captures captures, Context context) return Result.empty(); } - Set postFilterSymbols = postFilterAssignments.getSymbols(); + Set postFilterSymbols = postFilterAssignments.outputs(); // Remove inlined expressions from the underlying projection. newAssignments.putAll(projectNode.getAssignments().filter(symbol -> !postFilterSymbols.contains(symbol))); Map outputAssignments = new HashMap<>(); - outputAssignments.putAll(Assignments.identity(node.getOutputSymbols()).getMap()); + outputAssignments.putAll(Assignments.identity(node.getOutputSymbols()).assignments()); // Restore inlined symbols. - outputAssignments.putAll(postFilterAssignments.getMap()); + outputAssignments.putAll(postFilterAssignments.assignments()); return Result.ofPlanNode(new ProjectNode( context.getIdAllocator().getNextId(), diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjections.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjections.java index 1ffd76b006a7..d3c89d2c1bc1 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjections.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/InlineProjections.java @@ -161,7 +161,7 @@ private static Set extractInliningTargets(ProjectNode parent, ProjectNod Set childOutputSet = ImmutableSet.copyOf(child.getOutputSymbols()); Map dependencies = parent.getAssignments() - .getExpressions().stream() + .expressions().stream() .flatMap(expression -> SymbolsExtractor.extractAll(expression).stream()) .filter(childOutputSet::contains) .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MergePatternRecognitionNodes.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MergePatternRecognitionNodes.java index 6d3150415bec..3d71f2fad6e2 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MergePatternRecognitionNodes.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MergePatternRecognitionNodes.java @@ -137,7 +137,7 @@ public Result apply(PatternRecognitionNode node, Captures captures, Context cont // put prerequisite assignments in the source of merged node, // and the remaining assignments on top of merged node Assignments remainingAssignments = project.getAssignments() - .filter(symbol -> !prerequisites.getSymbols().contains(symbol)); + .filter(symbol -> !prerequisites.outputs().contains(symbol)); merged = (PatternRecognitionNode) merged.replaceChildren(ImmutableList.of(new ProjectNode( context.getIdAllocator().getNextId(), diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java index a310cdf7ce5a..54f68d00934b 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/MultipleDistinctAggregationToMarkDistinct.java @@ -72,8 +72,7 @@ public class MultipleDistinctAggregationToMarkDistinct public static boolean canUseMarkDistinct(AggregationNode aggregationNode) { - return hasNoDistinctWithFilterOrMask(aggregationNode) && - (hasMultipleDistincts(aggregationNode) || hasMixedDistinctAndNonDistincts(aggregationNode)); + return hasNoDistinctWithFilterOrMask(aggregationNode) && (hasMultipleDistincts(aggregationNode) || hasMixedDistinctAndNonDistincts(aggregationNode)); } private static boolean hasNoDistinctWithFilterOrMask(AggregationNode aggregationNode) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeDuplicateInsensitiveJoins.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeDuplicateInsensitiveJoins.java index c3f3ebf6c466..9008c9498a2a 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeDuplicateInsensitiveJoins.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeDuplicateInsensitiveJoins.java @@ -103,7 +103,7 @@ public Optional visitFilter(FilterNode node, Void context) @Override public Optional visitProject(ProjectNode node, Void context) { - boolean isDeterministic = node.getAssignments().getExpressions().stream() + boolean isDeterministic = node.getAssignments().expressions().stream() .allMatch(DeterminismEvaluator::isDeterministic); if (!isDeterministic) { // non-deterministic projections could be used in downstream filters which could diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeMixedDistinctAggregations.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeMixedDistinctAggregations.java index abfd1e0c3816..42677a106d73 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeMixedDistinctAggregations.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/OptimizeMixedDistinctAggregations.java @@ -88,11 +88,11 @@ public static boolean canUsePreAggregate(AggregationNode aggregationNode) { // single distinct can be supported in this rule, but it is already supported by SingleDistinctAggregationToGroupBy, which produces simpler plans (without group-id) return (hasMultipleDistincts(aggregationNode) || hasMixedDistinctAndNonDistincts(aggregationNode)) && - allDistinctAggregationsHaveSingleArgument(aggregationNode) && - noFilters(aggregationNode) && - noMasks(aggregationNode) && - !aggregationNode.hasOrderings() && - aggregationNode.getStep().equals(SINGLE); + allDistinctAggregationsHaveSingleArgument(aggregationNode) && + noFilters(aggregationNode) && + noMasks(aggregationNode) && + !aggregationNode.hasOrderings() && + aggregationNode.getStep().equals(SINGLE); } public static boolean hasMultipleDistincts(AggregationNode aggregationNode) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PreAggregateCaseAggregations.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PreAggregateCaseAggregations.java index 0de5602456ce..d54d7cc3e5de 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PreAggregateCaseAggregations.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PreAggregateCaseAggregations.java @@ -179,7 +179,7 @@ public Result apply(AggregationNode aggregationNode, Captures captures, Context context); AggregationNode preAggregation = createPreAggregation( preProjection, - preGroupingExpressions.getOutputs(), + preGroupingExpressions.outputs(), preAggregations, context); Map newProjectionSymbols = getNewProjectionSymbols(aggregations, context); @@ -247,7 +247,7 @@ private Map getNewProjectionSymbols(List groupingKeys, + Set groupingKeys, Map preAggregations, Context context) { diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ProjectOffPushDownRule.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ProjectOffPushDownRule.java index 07274390883c..21422f407b08 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ProjectOffPushDownRule.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/ProjectOffPushDownRule.java @@ -59,7 +59,7 @@ public Result apply(ProjectNode parent, Captures captures, Context context) { N targetNode = captures.get(targetCapture); - return pruneInputs(targetNode.getOutputSymbols(), parent.getAssignments().getExpressions()) + return pruneInputs(targetNode.getOutputSymbols(), parent.getAssignments().expressions()) .flatMap(prunedOutputs -> this.pushDownProjectOff(context, targetNode, prunedOutputs)) .map(newChild -> parent.replaceChildren(ImmutableList.of(newChild))) .map(Result::ofPlanNode) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughFilter.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughFilter.java index 6e1946e162ed..abaa5584d70d 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughFilter.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughFilter.java @@ -75,7 +75,7 @@ public Result apply(ProjectNode node, Captures captures, Rule.Context context) // Pushdown superset of dereference expressions from projections and filtering predicate List expressions = ImmutableList.builder() - .addAll(node.getAssignments().getExpressions()) + .addAll(node.getAssignments().expressions()) .add(filterNode.getPredicate()) .build(); @@ -90,7 +90,7 @@ public Result apply(ProjectNode node, Captures captures, Rule.Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughJoin.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughJoin.java index a9818e30707a..cd117652d802 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughJoin.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughJoin.java @@ -90,7 +90,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) // Consider dereferences in projections and join filter for pushdown ImmutableList.Builder expressionsBuilder = ImmutableList.builder(); - expressionsBuilder.addAll(projectNode.getAssignments().getExpressions()); + expressionsBuilder.addAll(projectNode.getAssignments().expressions()); joinNode.getFilter().ifPresent(expressionsBuilder::add); Set dereferences = extractRowSubscripts(expressionsBuilder.build(), false); @@ -114,7 +114,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); @@ -145,7 +145,7 @@ else if (joinNode.getRight().getOutputSymbols().contains(baseSymbol)) { PlanNode rightNode = createProjectNodeIfRequired(joinNode.getRight(), rightAssignments, context.getIdAllocator()); // Prepare new output symbols for join node - List referredSymbolsInAssignments = newAssignments.getExpressions().stream() + List referredSymbolsInAssignments = newAssignments.expressions().stream() .flatMap(expression -> extractAll(expression).stream()) .collect(toList()); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughProject.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughProject.java index 076054eada36..8747619fd690 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughProject.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughProject.java @@ -66,7 +66,7 @@ public Result apply(ProjectNode node, Captures captures, Context context) ProjectNode child = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(node.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(node.getAssignments().expressions(), false); // Exclude dereferences on symbols being synthesized within child dereferences = dereferences.stream() @@ -81,7 +81,7 @@ public Result apply(ProjectNode node, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughSemiJoin.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughSemiJoin.java index 0b5792fd1f34..b825a9d84825 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughSemiJoin.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughSemiJoin.java @@ -78,7 +78,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) SemiJoinNode semiJoinNode = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // All dereferences can be assumed on the symbols coming from source, since filteringSource output is not propagated, // and semiJoinOutput is of type boolean. We exclude pushdown of dereferences on sourceJoinSymbol. @@ -94,7 +94,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughUnnest.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughUnnest.java index a5a4a003448f..247ae276bf32 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughUnnest.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferenceThroughUnnest.java @@ -75,7 +75,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) UnnestNode unnestNode = captures.get(CHILD); // Extract dereferences for pushdown from project node's assignments - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // Only retain dereferences on replicate symbols dereferences = dereferences.stream() @@ -90,7 +90,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); @@ -114,7 +114,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) source, ImmutableList.builder() .addAll(unnestNode.getReplicateSymbols()) - .addAll(dereferenceAssignments.getSymbols()) + .addAll(dereferenceAssignments.outputs()) .build(), unnestNode.getMappings(), unnestNode.getOrdinalitySymbol(), diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughAssignUniqueId.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughAssignUniqueId.java index c40897f48c02..90d90bcb2dd9 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughAssignUniqueId.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughAssignUniqueId.java @@ -70,7 +70,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) AssignUniqueId assignUniqueId = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // We do not need to filter dereferences on idColumn symbol since it is supposed to be of BIGINT type. @@ -82,7 +82,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughLimit.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughLimit.java index e248caa4e727..d04ca70bb738 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughLimit.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughLimit.java @@ -75,7 +75,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) LimitNode limitNode = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // Exclude dereferences on symbols being used in tiesResolvingScheme and requiresPreSortedInputs Set excludedSymbols = ImmutableSet.builder() @@ -96,7 +96,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughMarkDistinct.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughMarkDistinct.java index 5d064f740658..79561e6dcd64 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughMarkDistinct.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughMarkDistinct.java @@ -75,7 +75,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) MarkDistinctNode markDistinctNode = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // Exclude dereferences on distinct symbols being used in markDistinctNode. We do not need to filter // dereferences on markerSymbol since it is supposed to be of boolean type. @@ -91,7 +91,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughRowNumber.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughRowNumber.java index 6ca0e9f576f2..5cadb82631cf 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughRowNumber.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughRowNumber.java @@ -75,7 +75,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) RowNumberNode rowNumberNode = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // Exclude dereferences on symbols being used in partitionBy dereferences = dereferences.stream() @@ -90,7 +90,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughSort.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughSort.java index 705ce82b20d2..2b4d20cb61b1 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughSort.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughSort.java @@ -75,7 +75,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) SortNode sortNode = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // Exclude dereferences on symbols used in ordering scheme to avoid replication of data dereferences = dereferences.stream() @@ -90,7 +90,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopN.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopN.java index d9ae3eacadac..0ea0712f99c9 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopN.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopN.java @@ -75,7 +75,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) TopNNode topNNode = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // Exclude dereferences on symbols being used in orderBy dereferences = dereferences.stream() @@ -90,7 +90,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopNRanking.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopNRanking.java index 77f1a4294ad0..ba310e5028a7 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopNRanking.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughTopNRanking.java @@ -78,7 +78,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) TopNRankingNode topNRankingNode = captures.get(CHILD); // Extract dereferences from project node assignments for pushdown - Set dereferences = extractRowSubscripts(projectNode.getAssignments().getExpressions(), false); + Set dereferences = extractRowSubscripts(projectNode.getAssignments().expressions(), false); // Exclude dereferences on symbols being used in partitionBy and orderBy DataOrganizationSpecification specification = topNRankingNode.getSpecification(); @@ -98,7 +98,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughWindow.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughWindow.java index f84c894bb35e..f2df38dc8276 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughWindow.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushDownDereferencesThroughWindow.java @@ -81,7 +81,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) // Extract dereferences for pushdown Set dereferences = extractRowSubscripts( ImmutableList.builder() - .addAll(projectNode.getAssignments().getExpressions()) + .addAll(projectNode.getAssignments().expressions()) // also include dereference projections used in window functions .addAll(windowNode.getWindowFunctions().values().stream() .flatMap(function -> function.getArguments().stream()) @@ -109,7 +109,7 @@ public Result apply(ProjectNode projectNode, Captures captures, Context context) Assignments dereferenceAssignments = Assignments.of(dereferences, context.getSymbolAllocator()); // Rewrite project node assignments using new symbols for dereference expressions - Map mappings = HashBiMap.create(dereferenceAssignments.getMap()) + Map mappings = HashBiMap.create(dereferenceAssignments.assignments()) .inverse() .entrySet().stream() .collect(toImmutableMap(Map.Entry::getKey, entry -> entry.getValue().toSymbolReference())); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushLimitThroughProject.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushLimitThroughProject.java index f40a2d798e4e..f924b1b869c6 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushLimitThroughProject.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushLimitThroughProject.java @@ -63,7 +63,7 @@ public Result apply(LimitNode parent, Captures captures, Context context) // Do not push down if the projection is made up of symbol references and exclusive dereferences. This prevents // undoing of PushDownDereferencesThroughLimit. We still push limit in the case of overlapping dereferences since // it enables PushDownDereferencesThroughLimit rule to push optimal dereferences. - Set projections = ImmutableSet.copyOf(projectNode.getAssignments().getExpressions()); + Set projections = ImmutableSet.copyOf(projectNode.getAssignments().expressions()); if (!extractRowSubscripts(projections, false).isEmpty() && exclusiveDereferences(projections)) { return Result.empty(); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoRowNumber.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoRowNumber.java index ac2a56acc5ea..92b158673f04 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoRowNumber.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoRowNumber.java @@ -102,7 +102,7 @@ public Result apply(FilterNode filter, Captures captures, Context context) RowNumberNode rowNumber = captures.get(ROW_NUMBER); Symbol rowNumberSymbol = rowNumber.getRowNumberSymbol(); - if (!project.getAssignments().getSymbols().contains(rowNumberSymbol)) { + if (!project.getAssignments().outputs().contains(rowNumberSymbol)) { return Result.empty(); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoWindow.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoWindow.java index 68af444a0823..b4715f108cec 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoWindow.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushPredicateThroughProjectIntoWindow.java @@ -113,7 +113,7 @@ public Result apply(FilterNode filter, Captures captures, Context context) WindowNode window = captures.get(WINDOW); Symbol rankingSymbol = getOnlyElement(window.getWindowFunctions().keySet()); - if (!project.getAssignments().getSymbols().contains(rankingSymbol)) { + if (!project.getAssignments().outputs().contains(rankingSymbol)) { return Result.empty(); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionIntoTableScan.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionIntoTableScan.java index 19268062eb55..1b3396068c0c 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionIntoTableScan.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionIntoTableScan.java @@ -99,7 +99,7 @@ public Result apply(ProjectNode project, Captures captures, Context context) // Extract translatable components from projection expressions. Prepare a mapping from these internal // expression nodes to corresponding ConnectorExpression translations. - Map, ConnectorExpression> partialTranslations = project.getAssignments().getMap().entrySet().stream() + Map, ConnectorExpression> partialTranslations = project.getAssignments().assignments().entrySet().stream() .flatMap(expression -> extractPartialTranslations( expression.getValue(), diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughExchange.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughExchange.java index ecc4ce3dfc1d..b8f9a8010d01 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughExchange.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughExchange.java @@ -178,7 +178,7 @@ public Result apply(ProjectNode project, Captures captures, Context context) private static boolean isSymbolToSymbolProjection(ProjectNode project) { - return project.getAssignments().getExpressions().stream().allMatch(Reference.class::isInstance); + return project.getAssignments().expressions().stream().allMatch(Reference.class::isInstance); } private static Map mapExchangeOutputToInput(ExchangeNode exchange, int sourceIndex) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughJoin.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughJoin.java index 661dec12e149..6644a669addc 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughJoin.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughJoin.java @@ -48,7 +48,7 @@ public static Optional pushProjectionThroughJoin( Lookup lookup, PlanNodeIdAllocator planNodeIdAllocator) { - if (!projectNode.getAssignments().getExpressions().stream().allMatch(DeterminismEvaluator::isDeterministic)) { + if (!projectNode.getAssignments().expressions().stream().allMatch(DeterminismEvaluator::isDeterministic)) { return Optional.empty(); } @@ -97,10 +97,10 @@ else if (rightChild.getOutputSymbols().containsAll(symbols)) { Assignments leftAssignments = leftAssignmentsBuilder.build(); Assignments rightAssignments = rightAssignmentsBuilder.build(); - List leftOutputSymbols = leftAssignments.getOutputs().stream() + List leftOutputSymbols = leftAssignments.outputs().stream() .filter(ImmutableSet.copyOf(projectNode.getOutputSymbols())::contains) .collect(toImmutableList()); - List rightOutputSymbols = rightAssignments.getOutputs().stream() + List rightOutputSymbols = rightAssignments.outputs().stream() .filter(ImmutableSet.copyOf(projectNode.getOutputSymbols())::contains) .collect(toImmutableList()); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughUnion.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughUnion.java index 699b696eff7d..76755a928001 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughUnion.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushProjectionThroughUnion.java @@ -90,7 +90,7 @@ public Result apply(ProjectNode parent, Captures captures, Context context) private static boolean nonTrivialProjection(ProjectNode project) { return !project.getAssignments() - .getExpressions().stream() + .expressions().stream() .allMatch(Reference.class::isInstance); } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNThroughProject.java b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNThroughProject.java index 2157f26c9036..77014d2c8b0f 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNThroughProject.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/iterative/rule/PushTopNThroughProject.java @@ -79,7 +79,7 @@ public Result apply(TopNNode parent, Captures captures, Context context) // Do not push down if the projection is made up of symbol references and exclusive dereferences. This prevents // undoing of PushDownDereferencesThroughTopN. We still push topN in the case of overlapping dereferences since // it enables PushDownDereferencesThroughTopN rule to push optimal dereferences. - Set projections = ImmutableSet.copyOf(projectNode.getAssignments().getExpressions()); + Set projections = ImmutableSet.copyOf(projectNode.getAssignments().expressions()); if (!extractRowSubscripts(projections, false).isEmpty() && exclusiveDereferences(projections)) { return Result.empty(); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java index cf8e85d3711f..73c69516737e 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddExchanges.java @@ -747,7 +747,15 @@ public PlanWithProperties visitTableExecute(TableExecuteNode node, PreferredProp { // Disable scale writers for partitioned data in case of Optimize since it can lead to small files and // not deterministic wrt to user provided min file size configuration. - boolean scaleWriters = node.getPartitioningScheme().isEmpty() && isScaleWriters(session); + boolean scaleWriters; + if (node.getPartitioningScheme().isPresent()) { + // Prefer partitioning by the execute node's partitioning scheme to attempt partitioning pushdown into the connector table scan + preferredProperties = PreferredProperties.partitioned(node.getPartitioningScheme().get().getPartitioning()); + scaleWriters = false; + } + else { + scaleWriters = isScaleWriters(session); + } return visitTableWriter(node, node.getPartitioningScheme(), node.getSource(), preferredProperties, node.getTarget(), scaleWriters); } @@ -866,12 +874,8 @@ public PlanWithProperties visitTableUpdate(TableUpdateNode node, PreferredProper @Override public PlanWithProperties visitExplainAnalyze(ExplainAnalyzeNode node, PreferredProperties preferredProperties) { - PlanWithProperties child = planChild(node, PreferredProperties.any()); - - // if the child is already a gathering exchange, don't add another - if ((child.getNode() instanceof ExchangeNode) && ((ExchangeNode) child.getNode()).getType() == ExchangeNode.Type.GATHER) { - return rebaseAndDeriveProperties(node, child); - } + // Same PreferredProperties as OutputNode + PlanWithProperties child = planChild(node, PreferredProperties.undistributed()); // Always add an exchange because ExplainAnalyze should be in its own stage child = withDerivedProperties( @@ -1530,7 +1534,7 @@ private boolean isNodePartitionedOn(ActualProperties properties, Collection computeIdentityTranslations(Assignments assignments) { Map outputToInput = new HashMap<>(); - for (Map.Entry assignment : assignments.getMap().entrySet()) { + for (Map.Entry assignment : assignments.assignments().entrySet()) { if (assignment.getValue() instanceof Reference) { outputToInput.put(assignment.getKey(), Symbol.from(assignment.getValue())); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java index 2860f28d5e98..0433d544eab3 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/AddLocalExchanges.java @@ -183,7 +183,7 @@ public PlanWithProperties visitProject(ProjectNode node, StreamPreferredProperti { // Special handling for trivial projections. Applies to identity and renaming projections, and constants // It might be extended to handle other low-cost projections. - if (node.getAssignments().getExpressions().stream().allMatch(expression -> expression instanceof Reference || expression instanceof Constant constant && constant.value() != null)) { + if (node.getAssignments().expressions().stream().allMatch(expression -> expression instanceof Reference || expression instanceof Constant constant && constant.value() != null)) { if (parentPreferences.isSingleStreamPreferred()) { // Do not enforce gathering exchange below project: // - if project's source is single stream, no exchanges will be added around project, diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/IndexJoinOptimizer.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/IndexJoinOptimizer.java index 3a28ab728da4..4f81a4489a33 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/IndexJoinOptimizer.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/IndexJoinOptimizer.java @@ -480,7 +480,7 @@ protected Map visitPlan(PlanNode node, Set lookupSymbols public Map visitProject(ProjectNode node, Set lookupSymbols) { // Map from output Symbols to source Symbols - Map directSymbolTranslationOutputMap = Maps.transformValues(Maps.filterValues(node.getAssignments().getMap(), Reference.class::isInstance), Symbol::from); + Map directSymbolTranslationOutputMap = Maps.transformValues(Maps.filterValues(node.getAssignments().assignments(), Reference.class::isInstance), Symbol::from); Map outputToSourceMap = lookupSymbols.stream() .filter(directSymbolTranslationOutputMap.keySet()::contains) .collect(toImmutableMap(identity(), directSymbolTranslationOutputMap::get)); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/MetadataQueryOptimizer.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/MetadataQueryOptimizer.java index 37a4fcbf394e..906683e83105 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/MetadataQueryOptimizer.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/MetadataQueryOptimizer.java @@ -195,7 +195,7 @@ private Optional findTableScan(PlanNode source) } else if (source instanceof ProjectNode project) { // verify projections are deterministic - if (!Iterables.all(project.getAssignments().getExpressions(), DeterminismEvaluator::isDeterministic)) { + if (!Iterables.all(project.getAssignments().expressions(), DeterminismEvaluator::isDeterministic)) { return Optional.empty(); } source = project.getSource(); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/OptimizerStats.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/OptimizerStats.java index e32e2d93d798..f3ea59f7cb21 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/OptimizerStats.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/OptimizerStats.java @@ -18,11 +18,11 @@ import org.weakref.jmx.Nested; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; public class OptimizerStats { - private final AtomicLong failures = new AtomicLong(); + private final LongAdder failures = new LongAdder(); private final TimeDistribution time = new TimeDistribution(TimeUnit.MICROSECONDS); public void record(long nanos) @@ -32,7 +32,7 @@ public void record(long nanos) public void recordFailure() { - failures.incrementAndGet(); + failures.increment(); } @Managed @@ -45,6 +45,6 @@ public TimeDistribution getTime() @Managed public long getFailures() { - return failures.get(); + return failures.sum(); } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PredicatePushDown.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PredicatePushDown.java index f66dc4d0fc83..4fd2c1e2a56e 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PredicatePushDown.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PredicatePushDown.java @@ -296,7 +296,7 @@ public PlanNode visitProject(ProjectNode node, RewriteContext contex .collect(Collectors.partitioningBy(expression -> isInliningCandidate(expression, node))); List inlinedDeterministicConjuncts = inlineConjuncts.get(true).stream() - .map(entry -> inlineSymbols(node.getAssignments().getMap(), entry)) + .map(entry -> inlineSymbols(node.getAssignments().assignments(), entry)) .map(conjunct -> canonicalizeExpression(conjunct, plannerContext)) // normalize expressions to a form that unwrapCasts understands .map(conjunct -> unwrapCasts(session, plannerContext, conjunct)) .collect(Collectors.toList()); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java index c408eca3391e..8f64190b7512 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/PropertyDerivations.java @@ -761,9 +761,9 @@ public ActualProperties visitProject(ProjectNode node, List in { ActualProperties properties = Iterables.getOnlyElement(inputProperties); - Map identities = computeIdentityTranslations(node.getAssignments().getMap()); + Map identities = computeIdentityTranslations(node.getAssignments().assignments()); - ActualProperties translatedProperties = properties.translate(column -> Optional.ofNullable(identities.get(column)), expression -> rewriteExpression(node.getAssignments().getMap(), expression)); + ActualProperties translatedProperties = properties.translate(column -> Optional.ofNullable(identities.get(column)), expression -> rewriteExpression(node.getAssignments().assignments(), expression)); // Extract additional constants Map constants = new HashMap<>(); diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/StreamPropertyDerivations.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/StreamPropertyDerivations.java index f26d3bf01a4a..6884394b9f7d 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/StreamPropertyDerivations.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/StreamPropertyDerivations.java @@ -405,7 +405,7 @@ public StreamProperties visitProject(ProjectNode node, List in StreamProperties properties = Iterables.getOnlyElement(inputProperties); // We can describe properties in terms of inputs that are projected unmodified (i.e., identity projections) - Map identities = computeIdentityTranslations(node.getAssignments().getMap()); + Map identities = computeIdentityTranslations(node.getAssignments().assignments()); return properties.translate(column -> Optional.ofNullable(identities.get(column))); } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/SymbolMapper.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/SymbolMapper.java index 406b1412c327..afde19899a81 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/SymbolMapper.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/SymbolMapper.java @@ -58,6 +58,7 @@ import io.trino.sql.planner.rowpattern.ValuePointer; import io.trino.sql.planner.rowpattern.ir.IrLabel; +import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -140,7 +141,7 @@ public List map(List symbols) .collect(toImmutableList()); } - public List mapAndDistinct(List symbols) + public List mapAndDistinct(Collection symbols) { return symbols.stream() .map(this::map) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/UnaliasSymbolReferences.java b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/UnaliasSymbolReferences.java index 00160c2ae78e..606d37bd6346 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -687,12 +687,11 @@ public PlanAndMappings visitSimpleTableExecuteNode(SimpleTableExecuteNode node, { Map mapping = new HashMap<>(context.getCorrelationMapping()); SymbolMapper mapper = symbolMapper(mapping); - Symbol newOutput = mapper.map(node.getOutput()); return new PlanAndMappings( new SimpleTableExecuteNode( node.getId(), - newOutput, + mapper.map(node.getOutputSymbols()), node.getExecuteHandle()), mapping); } @@ -847,7 +846,7 @@ public PlanAndMappings visitProject(ProjectNode node, UnaliasContext context) // Those symbols are supposed to represent constant semantics throughout the plan. Assignments assignments = node.getAssignments(); - Set newlyAssignedSymbols = assignments.filter(output -> !assignments.isIdentity(output)).getSymbols(); + Set newlyAssignedSymbols = assignments.filter(output -> !assignments.isIdentity(output)).outputs(); Set symbolsInSourceMapping = ImmutableSet.builder() .addAll(rewrittenSource.getMappings().keySet()) .addAll(rewrittenSource.getMappings().values()) diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/plan/AggregationNode.java b/core/trino-main/src/main/java/io/trino/sql/planner/plan/AggregationNode.java index e8c7c4af5083..633942cc2247 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/plan/AggregationNode.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/plan/AggregationNode.java @@ -31,6 +31,7 @@ import io.trino.sql.planner.Symbol; import io.trino.type.FunctionType; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -292,10 +293,10 @@ public boolean isStreamable() public static GroupingSetDescriptor globalAggregation() { - return singleGroupingSet(ImmutableList.of()); + return singleGroupingSet(ImmutableSet.of()); } - public static GroupingSetDescriptor singleGroupingSet(List groupingKeys) + public static GroupingSetDescriptor singleGroupingSet(Collection groupingKeys) { Set globalGroupingSets; if (groupingKeys.isEmpty()) { @@ -308,7 +309,7 @@ public static GroupingSetDescriptor singleGroupingSet(List groupingKeys) return new GroupingSetDescriptor(groupingKeys, 1, globalGroupingSets); } - public static GroupingSetDescriptor groupingSets(List groupingKeys, int groupingSetCount, Set globalGroupingSets) + public static GroupingSetDescriptor groupingSets(Collection groupingKeys, int groupingSetCount, Set globalGroupingSets) { return new GroupingSetDescriptor(groupingKeys, groupingSetCount, globalGroupingSets); } @@ -321,7 +322,7 @@ public static class GroupingSetDescriptor @JsonCreator public GroupingSetDescriptor( - @JsonProperty("groupingKeys") List groupingKeys, + @JsonProperty("groupingKeys") Collection groupingKeys, @JsonProperty("groupingSetCount") int groupingSetCount, @JsonProperty("globalGroupingSets") Set globalGroupingSets) { diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/plan/Assignments.java b/core/trino-main/src/main/java/io/trino/sql/planner/plan/Assignments.java index 1aa78457c369..b48d82897ac2 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/plan/Assignments.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/plan/Assignments.java @@ -15,9 +15,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; +import com.google.common.collect.ImmutableSet; import io.trino.sql.ir.Expression; import io.trino.sql.ir.Reference; import io.trino.sql.planner.Symbol; @@ -25,20 +24,18 @@ import java.util.Collection; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Predicate; -import java.util.stream.Collector; import static com.google.common.base.Preconditions.checkArgument; import static java.util.Arrays.asList; import static java.util.Objects.requireNonNull; -public class Assignments +public record Assignments(@JsonProperty("assignments") Map assignments) { public static Builder builder() { @@ -79,7 +76,7 @@ public static Assignments of(Symbol symbol1, Expression expression1, Symbol symb public static Assignments of(Collection expressions, SymbolAllocator symbolAllocator) { - Assignments.Builder assignments = Assignments.builder(); + Builder assignments = Assignments.builder(); for (Expression expression : expressions) { assignments.put(symbolAllocator.newSymbol(expression), expression); @@ -88,30 +85,24 @@ public static Assignments of(Collection expressions, Symbo return assignments.build(); } - private final Map assignments; - @JsonCreator public Assignments(@JsonProperty("assignments") Map assignments) { this.assignments = ImmutableMap.copyOf(requireNonNull(assignments, "assignments is null")); } - public List getOutputs() - { - return ImmutableList.copyOf(assignments.keySet()); - } - - @JsonProperty("assignments") - public Map getMap() + public Set outputs() { - return assignments; + return ImmutableSet.copyOf(assignments.keySet()); } public Assignments rewrite(Function rewrite) { - return assignments.entrySet().stream() - .map(entry -> Maps.immutableEntry(entry.getKey(), rewrite.apply(entry.getValue()))) - .collect(toAssignments()); + ImmutableMap.Builder builder = ImmutableMap.builderWithExpectedSize(assignments.size()); + for (Entry entry : assignments.entrySet()) { + builder.put(entry.getKey(), rewrite.apply(entry.getValue())); + } + return new Assignments(builder.buildOrThrow()); } public Assignments filter(Collection symbols) @@ -121,21 +112,23 @@ public Assignments filter(Collection symbols) public Assignments filter(Predicate predicate) { - return assignments.entrySet().stream() - .filter(entry -> predicate.test(entry.getKey())) - .collect(toAssignments()); + ImmutableMap.Builder builder = ImmutableMap.builder(); + for (Entry entry : assignments.entrySet()) { + if (predicate.test(entry.getKey())) { + builder.put(entry.getKey(), entry.getValue()); + } + } + return new Assignments(builder.buildOrThrow()); } public boolean isIdentity(Symbol output) { - Expression expression = assignments.get(output); - - return expression instanceof Reference reference && reference.name().equals(output.name()); + return assignments.get(output) instanceof Reference reference && reference.name().equals(output.name()); } public boolean isIdentity() { - for (Map.Entry entry : assignments.entrySet()) { + for (Entry entry : assignments.entrySet()) { Expression expression = entry.getValue(); Symbol symbol = entry.getKey(); if (!(expression instanceof Reference reference && reference.name().equals(symbol.name()))) { @@ -145,28 +138,11 @@ public boolean isIdentity() return true; } - private Collector, Builder, Assignments> toAssignments() - { - return Collector.of( - Assignments::builder, - (builder, entry) -> builder.put(entry.getKey(), entry.getValue()), - (left, right) -> { - left.putAll(right.build()); - return left; - }, - Assignments.Builder::build); - } - - public Collection getExpressions() + public Collection expressions() { return assignments.values(); } - public Set getSymbols() - { - return assignments.keySet(); - } - public Set> entrySet() { return assignments.entrySet(); @@ -184,7 +160,7 @@ public int size() public boolean isEmpty() { - return size() == 0; + return assignments.isEmpty(); } public void forEach(BiConsumer consumer) @@ -192,46 +168,12 @@ public void forEach(BiConsumer consumer) assignments.forEach(consumer); } - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Assignments that = (Assignments) o; - - return assignments.equals(that.assignments); - } - - @Override - public int hashCode() + public record Assignment(Symbol output, Expression expression) { - return assignments.hashCode(); - } - - public static class Assignment - { - private final Symbol output; - private final Expression expression; - - public Assignment(Symbol output, Expression expression) - { - this.output = requireNonNull(output, "output is null"); - this.expression = requireNonNull(expression, "expression is null"); - } - - public Symbol getOutput() - { - return output; - } - - public Expression getExpression() + public Assignment { - return expression; + requireNonNull(output, "output is null"); + requireNonNull(expression, "expression is null"); } } @@ -241,7 +183,7 @@ public static class Builder public Builder putAll(Assignments assignments) { - return putAll(assignments.getMap()); + return putAll(assignments.assignments()); } public Builder putAll(Map assignments) @@ -270,6 +212,12 @@ public Builder put(Entry assignment) return this; } + public Builder add(Assignment assignment) + { + put(assignment.output(), assignment.expression()); + return this; + } + public Builder putIdentities(Iterable symbols) { for (Symbol symbol : symbols) { @@ -288,11 +236,5 @@ public Assignments build() { return new Assignments(assignments); } - - public Builder add(Assignment assignment) - { - put(assignment.getOutput(), assignment.getExpression()); - return this; - } } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/plan/ProjectNode.java b/core/trino-main/src/main/java/io/trino/sql/planner/plan/ProjectNode.java index 853a5cf53abd..47b15a5d59e1 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/plan/ProjectNode.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/plan/ProjectNode.java @@ -49,7 +49,7 @@ public ProjectNode(@JsonProperty("id") PlanNodeId id, @Override public List getOutputSymbols() { - return assignments.getOutputs(); + return ImmutableList.copyOf(assignments.outputs()); } @JsonProperty diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/plan/SimpleTableExecuteNode.java b/core/trino-main/src/main/java/io/trino/sql/planner/plan/SimpleTableExecuteNode.java index cba07cbd5866..80b3f643ff29 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/plan/SimpleTableExecuteNode.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/plan/SimpleTableExecuteNode.java @@ -27,17 +27,17 @@ public class SimpleTableExecuteNode extends PlanNode { - private final Symbol output; + private final List outputs; private final TableExecuteHandle executeHandle; @JsonCreator public SimpleTableExecuteNode( @JsonProperty("id") PlanNodeId id, - @JsonProperty("output") Symbol output, + @JsonProperty("outputs") List outputs, @JsonProperty("executeHandle") TableExecuteHandle executeHandle) { super(id); - this.output = requireNonNull(output, "output is null"); + this.outputs = ImmutableList.copyOf(requireNonNull(outputs, "outputs is null")); this.executeHandle = requireNonNull(executeHandle, "executeHandle is null"); } @@ -48,16 +48,11 @@ public List getSources() return ImmutableList.of(); } + @JsonProperty("outputs") @Override public List getOutputSymbols() { - return ImmutableList.of(output); - } - - @JsonProperty - public Symbol getOutput() - { - return output; + return outputs; } @Override diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanPrinter.java b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanPrinter.java index 73bb416e4404..3704e09a824f 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanPrinter.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/planprinter/PlanPrinter.java @@ -31,7 +31,6 @@ import io.trino.cost.PlanNodeStatsAndCostSummary; import io.trino.cost.PlanNodeStatsEstimate; import io.trino.cost.StatsAndCosts; -import io.trino.execution.DistributionSnapshot; import io.trino.execution.QueryStats; import io.trino.execution.StageInfo; import io.trino.execution.StageStats; @@ -42,6 +41,7 @@ import io.trino.metadata.Metadata; import io.trino.metadata.ResolvedFunction; import io.trino.metadata.TableHandle; +import io.trino.plugin.base.metrics.DistributionSnapshot; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.expression.FunctionName; import io.trino.spi.function.CatalogSchemaFunctionName; @@ -428,7 +428,7 @@ public static String textDistributedPlan( Anonymizer anonymizer, NodeVersion version) { - return textDistributedPlan(stages.getStages(), queryStats, valuePrinter, verbose, anonymizer, version); + return textDistributedPlan(stages.getSubStagesDeepTopological(stages.getOutputStageId(), true), queryStats, valuePrinter, verbose, anonymizer, version); } public static String textDistributedPlan( @@ -465,11 +465,12 @@ public static String textDistributedPlan(List stages, QueryStats quer .collect(toImmutableMap(DynamicFilterDomainStats::getDynamicFilterId, identity())); builder.append(format("Trino version: %s\n", version)); - builder.append(format("Queued: %s, Analysis: %s, Planning: %s, Execution: %s\n", + builder.append(format("Queued: %s, Analysis: %s, Planning: %s, Execution: %s, Finishing: %s\n", queryStats.getQueuedTime().convertToMostSuccinctTimeUnit(), queryStats.getAnalysisTime().convertToMostSuccinctTimeUnit(), queryStats.getPlanningTime().convertToMostSuccinctTimeUnit(), - queryStats.getExecutionTime().convertToMostSuccinctTimeUnit())); + queryStats.getExecutionTime().convertToMostSuccinctTimeUnit(), + queryStats.getFinishingTime().convertToMostSuccinctTimeUnit())); for (StageInfo stageInfo : stages) { if (stageInfo.getPlan() == null) { @@ -2007,7 +2008,7 @@ private Void processChildren(PlanNode node, Context context) private void printAssignments(NodeRepresentation nodeOutput, Assignments assignments) { - for (Entry entry : assignments.getMap().entrySet()) { + for (Entry entry : assignments.assignments().entrySet()) { if (entry.getValue() instanceof Reference && ((Reference) entry.getValue()).name().equals(entry.getKey().name())) { // skip identity assignments continue; diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/sanity/TableExecuteStructureValidator.java b/core/trino-main/src/main/java/io/trino/sql/planner/sanity/TableExecuteStructureValidator.java index 3b855cc263f6..389c1adfab13 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/sanity/TableExecuteStructureValidator.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/sanity/TableExecuteStructureValidator.java @@ -18,6 +18,7 @@ import io.trino.sql.PlannerContext; import io.trino.sql.planner.AdaptivePlanner; import io.trino.sql.planner.plan.ExchangeNode; +import io.trino.sql.planner.plan.ExplainAnalyzeNode; import io.trino.sql.planner.plan.OutputNode; import io.trino.sql.planner.plan.PlanNode; import io.trino.sql.planner.plan.ProjectNode; @@ -65,6 +66,7 @@ private boolean isAllowedNode(PlanNode node) || node instanceof TableExecuteNode || node instanceof OutputNode || node instanceof ExchangeNode - || node instanceof TableFinishNode; + || node instanceof TableFinishNode + || node instanceof ExplainAnalyzeNode; } } diff --git a/core/trino-main/src/main/java/io/trino/sql/planner/sanity/ValidateDependenciesChecker.java b/core/trino-main/src/main/java/io/trino/sql/planner/sanity/ValidateDependenciesChecker.java index c224979db597..9d6f1ccde763 100644 --- a/core/trino-main/src/main/java/io/trino/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/core/trino-main/src/main/java/io/trino/sql/planner/sanity/ValidateDependenciesChecker.java @@ -448,7 +448,7 @@ public Void visitProject(ProjectNode node, Set boundSymbols) source.accept(this, boundSymbols); // visit child Set inputs = createInputs(source, boundSymbols); - for (Expression expression : node.getAssignments().getExpressions()) { + for (Expression expression : node.getAssignments().expressions()) { Set dependencies = extractUnique(expression); checkDependencies(inputs, dependencies, "Invalid node. Expression dependencies (%s) not in source plan output (%s)", dependencies, inputs); } diff --git a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java index 9028c0b61767..e41574a6fa7c 100644 --- a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java +++ b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java @@ -384,7 +384,8 @@ private PlanTester(Session defaultSession, int nodeCountForStats) globalFunctionCatalog, languageFunctionManager, tableFunctionRegistry, - typeManager); + typeManager, + catalogManager); typeRegistry.addType(new JsonPath2016Type(new TypeDeserializer(typeManager), blockEncodingSerde)); this.joinCompiler = new JoinCompiler(typeOperators); this.hashStrategyCompiler = new FlatHashStrategyCompiler(typeOperators); diff --git a/core/trino-main/src/main/java/io/trino/testing/StandaloneQueryRunner.java b/core/trino-main/src/main/java/io/trino/testing/StandaloneQueryRunner.java index 4dc2acea9f2b..464b1e8e2c71 100644 --- a/core/trino-main/src/main/java/io/trino/testing/StandaloneQueryRunner.java +++ b/core/trino-main/src/main/java/io/trino/testing/StandaloneQueryRunner.java @@ -20,6 +20,7 @@ import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; import io.trino.Session; +import io.trino.connector.ConnectorServicesProvider; import io.trino.cost.StatsCalculator; import io.trino.execution.FailureInjector.InjectedFailureType; import io.trino.execution.QueryManagerConfig; @@ -88,6 +89,7 @@ public StandaloneQueryRunner(Session defaultSession, Consumer executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) { Span span = startSpan("executeTableExecute", tableExecuteHandle); try (var _ = scopedSpan(span)) { - delegate.executeTableExecute(session, tableExecuteHandle); + return delegate.executeTableExecute(session, tableExecuteHandle); } } @@ -252,6 +253,14 @@ public Optional getInfo(ConnectorSession session, ConnectorTableHandle t } } + @Override + public Metrics getMetrics(ConnectorSession session) + { + // Do not trace getMetrics as this is expected to be a quick local jvm call, + // and adding this span would only obfuscate the trace + return delegate.getMetrics(session); + } + @Override public List listTables(ConnectorSession session, Optional schemaName) { @@ -578,6 +587,15 @@ public Optional getInsertLayout(ConnectorSession session, } } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean tableReplace) + { + Span span = startSpan("getStatisticsCollectionMetadataForWrite", tableMetadata.getTable()); + try (var _ = scopedSpan(span)) { + return delegate.getStatisticsCollectionMetadataForWrite(session, tableMetadata, tableReplace); + } + } + @Override public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) { diff --git a/core/trino-main/src/main/java/io/trino/tracing/TracingMetadata.java b/core/trino-main/src/main/java/io/trino/tracing/TracingMetadata.java index 42e15adf8ffe..a5423a180cb6 100644 --- a/core/trino-main/src/main/java/io/trino/tracing/TracingMetadata.java +++ b/core/trino-main/src/main/java/io/trino/tracing/TracingMetadata.java @@ -60,6 +60,7 @@ import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ColumnPosition; import io.trino.spi.connector.ConnectorCapabilities; +import io.trino.spi.connector.ConnectorName; import io.trino.spi.connector.ConnectorOutputMetadata; import io.trino.spi.connector.ConnectorTableMetadata; import io.trino.spi.connector.Constraint; @@ -96,6 +97,7 @@ import io.trino.spi.function.FunctionMetadata; import io.trino.spi.function.LanguageFunction; import io.trino.spi.function.OperatorType; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.FunctionAuthorization; import io.trino.spi.security.GrantInfo; @@ -236,11 +238,11 @@ public void finishTableExecute(Session session, TableExecuteHandle handle, Colle } @Override - public void executeTableExecute(Session session, TableExecuteHandle handle) + public Map executeTableExecute(Session session, TableExecuteHandle handle) { Span span = startSpan("executeTableExecute", handle); try (var _ = scopedSpan(span)) { - delegate.executeTableExecute(session, handle); + return delegate.executeTableExecute(session, handle); } } @@ -283,6 +285,14 @@ public Optional getInfo(Session session, TableHandle handle) } } + @Override + public Metrics getMetrics(Session session, String catalogName) + { + // Do not trace getMetrics as this is expected to be a quick local jvm call, + // and adding this span would only obfuscate the trace + return delegate.getMetrics(session, catalogName); + } + @Override public CatalogSchemaTableName getTableName(Session session, TableHandle tableHandle) { @@ -373,6 +383,24 @@ public List listRelationComments(Session session, Strin } } + @Override + public void createCatalog(Session session, CatalogName catalog, ConnectorName connectorName, Map properties, boolean notExists) + { + Span span = startSpan("createCatalog", catalog); + try (var _ = scopedSpan(span)) { + delegate.createCatalog(session, catalog, connectorName, properties, notExists); + } + } + + @Override + public void dropCatalog(Session session, CatalogName catalog, boolean cascade) + { + Span span = startSpan("dropCatalog", catalog); + try (var _ = scopedSpan(span)) { + delegate.dropCatalog(session, catalog, cascade); + } + } + @Override public void createSchema(Session session, CatalogSchemaName schema, Map properties, TrinoPrincipal principal) { @@ -611,11 +639,11 @@ public Optional getInsertLayout(Session session, TableHandle target } @Override - public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata) + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata, boolean tableReplace) { Span span = startSpan("getStatisticsCollectionMetadataForWrite", catalogHandle.getCatalogName().toString(), tableMetadata); try (var _ = scopedSpan(span)) { - return delegate.getStatisticsCollectionMetadataForWrite(session, catalogHandle, tableMetadata); + return delegate.getStatisticsCollectionMetadataForWrite(session, catalogHandle, tableMetadata, tableReplace); } } @@ -849,6 +877,15 @@ public List listCatalogs(Session session) } } + @Override + public List listActiveCatalogs(Session session) + { + Span span = startSpan("listActiveCatalogs"); + try (var _ = scopedSpan(span)) { + return delegate.listActiveCatalogs(session); + } + } + @Override public List listViews(Session session, QualifiedTablePrefix prefix) { diff --git a/core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java b/core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java index 64b9ea176eee..0944401dd61e 100644 --- a/core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java +++ b/core/trino-main/src/main/java/io/trino/transaction/InMemoryTransactionManager.java @@ -423,7 +423,7 @@ private synchronized List getActiveCatalogs() .distinct() .map(key -> registeredCatalogs.getOrDefault(key, Optional.empty())) .flatMap(Optional::stream) - .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.isLoaded())) + .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.getCatalogStatus())) .collect(toImmutableList()); } @@ -436,7 +436,7 @@ private synchronized List listCatalogs() return registeredCatalogs.values().stream() .filter(Optional::isPresent) .map(Optional::get) - .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.isLoaded())) + .map(catalog -> new CatalogInfo(catalog.getCatalogName().toString(), catalog.getCatalogHandle(), catalog.getConnectorName(), catalog.getCatalogStatus())) .collect(toImmutableList()); } diff --git a/core/trino-main/src/main/java/io/trino/type/FunctionType.java b/core/trino-main/src/main/java/io/trino/type/FunctionType.java index 531ac3186471..1070829d23b5 100644 --- a/core/trino-main/src/main/java/io/trino/type/FunctionType.java +++ b/core/trino-main/src/main/java/io/trino/type/FunctionType.java @@ -187,12 +187,6 @@ public Object getObjectValue(Block block, int position) throw new UnsupportedOperationException(); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - throw new UnsupportedOperationException(); - } - @Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries, int expectedBytesPerEntry) { diff --git a/core/trino-main/src/main/java/io/trino/type/IpAddressType.java b/core/trino-main/src/main/java/io/trino/type/IpAddressType.java index d652bc99e1a8..aee1f0953355 100644 --- a/core/trino-main/src/main/java/io/trino/type/IpAddressType.java +++ b/core/trino-main/src/main/java/io/trino/type/IpAddressType.java @@ -126,25 +126,6 @@ public Object getObjectValue(Block block, int position) } } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - appendTo( - (Int128ArrayBlock) block.getUnderlyingValueBlock(), - block.getUnderlyingValuePosition(position), - (Int128ArrayBlockBuilder) blockBuilder); - } - - private void appendTo(Int128ArrayBlock block, int position, Int128ArrayBlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - blockBuilder.writeInt128(block.getInt128High(position), block.getInt128Low(position)); - } - } - @Override public void writeSlice(BlockBuilder blockBuilder, Slice value) { diff --git a/core/trino-main/src/main/java/io/trino/type/UnknownType.java b/core/trino-main/src/main/java/io/trino/type/UnknownType.java index d1b8e70a5c04..04a8ce3349ed 100644 --- a/core/trino-main/src/main/java/io/trino/type/UnknownType.java +++ b/core/trino-main/src/main/java/io/trino/type/UnknownType.java @@ -108,12 +108,6 @@ public Object getObjectValue(Block block, int position) return null; } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - blockBuilder.appendNull(); - } - @Override public boolean getBoolean(Block block, int position) { diff --git a/core/trino-main/src/main/java/io/trino/util/MergeSortedPages.java b/core/trino-main/src/main/java/io/trino/util/MergeSortedPages.java index 10f5747f2789..1b685a74c0f8 100644 --- a/core/trino-main/src/main/java/io/trino/util/MergeSortedPages.java +++ b/core/trino-main/src/main/java/io/trino/util/MergeSortedPages.java @@ -121,7 +121,7 @@ private static WorkProcessor buildPage( Page page = pageBuilder.build(); pageBuilder.reset(); if (!finished) { - pageWithPosition.appendTo(pageBuilder, outputChannels, outputTypes); + pageWithPosition.appendTo(pageBuilder, outputChannels); } if (updateMemoryAfterEveryPosition) { @@ -131,7 +131,7 @@ private static WorkProcessor buildPage( return TransformationState.ofResult(page, !finished); } - pageWithPosition.appendTo(pageBuilder, outputChannels, outputTypes); + pageWithPosition.appendTo(pageBuilder, outputChannels); if (updateMemoryAfterEveryPosition) { memoryContext.setBytes(pageBuilder.getRetainedSizeInBytes()); @@ -186,13 +186,12 @@ public int getPosition() return position; } - public void appendTo(PageBuilder pageBuilder, List outputChannels, List outputTypes) + public void appendTo(PageBuilder pageBuilder, List outputChannels) { pageBuilder.declarePosition(); for (int i = 0; i < outputChannels.size(); i++) { - Type type = outputTypes.get(i); Block block = page.getBlock(outputChannels.get(i)); - type.appendTo(block, position, pageBuilder.getBlockBuilder(i)); + pageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } } } diff --git a/core/trino-main/src/test/java/io/trino/BenchmarkPagesIndexPageSorter.java b/core/trino-main/src/test/java/io/trino/BenchmarkPagesIndexPageSorter.java index c4021a505e93..c1d6e056327b 100644 --- a/core/trino-main/src/test/java/io/trino/BenchmarkPagesIndexPageSorter.java +++ b/core/trino-main/src/test/java/io/trino/BenchmarkPagesIndexPageSorter.java @@ -33,6 +33,7 @@ import org.openjdk.jmh.runner.RunnerException; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; @@ -55,8 +56,13 @@ public class BenchmarkPagesIndexPageSorter public int runBenchmark(BenchmarkData data) { PageSorter pageSorter = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false)); - long[] addresses = pageSorter.sort(data.types, data.pages, data.sortChannels, nCopies(data.sortChannels.size(), ASC_NULLS_FIRST), 10_000); - return addresses.length; + Iterator outputPages = pageSorter.sort(data.types, data.pages, data.sortChannels, nCopies(data.sortChannels.size(), ASC_NULLS_FIRST), 10_000); + int totalPositions = 0; + while (outputPages.hasNext()) { + Page page = outputPages.next(); + totalPositions += page.getPositionCount(); + } + return totalPositions; } private static List createPages(int pageCount, int channelCount, Type type) diff --git a/core/trino-main/src/test/java/io/trino/TestPagesIndexPageSorter.java b/core/trino-main/src/test/java/io/trino/TestPagesIndexPageSorter.java index 38140158e6f9..8b0530428379 100644 --- a/core/trino-main/src/test/java/io/trino/TestPagesIndexPageSorter.java +++ b/core/trino-main/src/test/java/io/trino/TestPagesIndexPageSorter.java @@ -18,7 +18,6 @@ import io.trino.operator.PagesIndex; import io.trino.operator.PagesIndexPageSorter; import io.trino.spi.Page; -import io.trino.spi.PageBuilder; import io.trino.spi.connector.SortOrder; import io.trino.spi.type.Type; import io.trino.testing.MaterializedResult; @@ -149,29 +148,10 @@ public void testPageSorterForceExpansion() private static void assertSorted(List inputPages, List expectedPages, List types, List sortChannels, List sortOrders, int expectedPositions) { - long[] sortedAddresses = sorter.sort(types, inputPages, sortChannels, sortOrders, expectedPositions); - List outputPages = createOutputPages(types, inputPages, sortedAddresses); + List outputPages = ImmutableList.copyOf(sorter.sort(types, inputPages, sortChannels, sortOrders, expectedPositions)); MaterializedResult expected = toMaterializedResult(TEST_SESSION, types, expectedPages); MaterializedResult actual = toMaterializedResult(TEST_SESSION, types, outputPages); assertThat(actual.getMaterializedRows()).isEqualTo(expected.getMaterializedRows()); } - - private static List createOutputPages(List types, List inputPages, long[] sortedAddresses) - { - PageBuilder pageBuilder = new PageBuilder(types); - pageBuilder.reset(); - for (long address : sortedAddresses) { - int index = sorter.decodePageIndex(address); - int position = sorter.decodePositionIndex(address); - - Page page = inputPages.get(index); - for (int i = 0; i < types.size(); i++) { - Type type = types.get(i); - type.appendTo(page.getBlock(i), position, pageBuilder.getBlockBuilder(i)); - } - pageBuilder.declarePosition(); - } - return ImmutableList.of(pageBuilder.build()); - } } diff --git a/core/trino-main/src/test/java/io/trino/block/BenchmarkBlockBuilder.java b/core/trino-main/src/test/java/io/trino/block/BenchmarkBlockBuilder.java new file mode 100644 index 000000000000..74460d312cd6 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/block/BenchmarkBlockBuilder.java @@ -0,0 +1,154 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.block; + +import io.airlift.slice.Slice; +import io.airlift.slice.Slices; +import io.trino.spi.block.Block; +import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.ValueBlock; +import io.trino.spi.type.Type; +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Random; + +import static io.trino.jmh.Benchmarks.benchmark; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +@State(Scope.Thread) +@OutputTimeUnit(NANOSECONDS) +@Fork(1) +@Warmup(iterations = 5, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkBlockBuilder +{ + public static final int POSITIONS = 1024; + public static final int MAX_STRING_LENGTH = 64; + + @Benchmark + public Block benchmarkAppendRange(BlockData data) + { + BlockBuilder builder = data.trinoType.createBlockBuilder(null, POSITIONS); + // Append twice to simulate "merging" two adjacent blocks + builder.appendRange(data.valueBlock, data.offset, data.length); + builder.appendRange(data.valueBlock, data.offset, data.length); + return builder.build(); + } + + @State(Scope.Thread) + public static class BlockData + { + @Param({"VARCHAR", "BIGINT"}) + private String type = "BIGINT"; + private Type trinoType; + @Param({"0", "0.1", "0.25", "0.5", "0.75", "0.9", "1.0"}) + private float nullRate = 0.5f; + private ValueBlock valueBlock; + private int offset; + private int length; + private Random random = new Random(1024); + + @Setup + public void setup() + { + offset = 1; + length = POSITIONS; + switch (this.type) { + case "VARCHAR" -> { + trinoType = VARCHAR; + valueBlock = createVarcharBlock(random, offset, length, nullRate); + } + case "BIGINT" -> { + trinoType = BIGINT; + valueBlock = createBigintBlock(random, offset, length, nullRate); + } + default -> throw new IllegalArgumentException("Unsupported type: " + this.type); + } + } + + private static ValueBlock createBigintBlock(Random random, int offset, int length, float nullRate) + { + BlockBuilder builder = BIGINT.createBlockBuilder(null, offset + length); + int position = 0; + // Fill the beginning range with nulls to prevent the isNull array from being suppressed + for (; position < offset; position++) { + builder.appendNull(); + } + for (; position < offset + length; position++) { + if (random.nextFloat() <= nullRate) { + builder.appendNull(); + } + else { + BIGINT.writeLong(builder, random.nextLong()); + } + } + return builder.buildValueBlock(); + } + + private static ValueBlock createVarcharBlock(Random random, int offset, int length, float nullRate) + { + BlockBuilder builder = VARCHAR.createBlockBuilder(null, offset + length); + int position = 0; + // Fill the beginning range with nulls to prevent the isNull array from being suppressed + for (; position < offset; position++) { + builder.appendNull(); + } + for (; position < offset + length; position++) { + if (random.nextFloat() <= nullRate) { + builder.appendNull(); + } + else { + VARCHAR.writeSlice(builder, createRandomAsciiString(random)); + } + } + return builder.buildValueBlock(); + } + + private static Slice createRandomAsciiString(Random random) + { + byte[] data = new byte[random.nextInt(MAX_STRING_LENGTH)]; + random.nextBytes(data); + return Slices.wrappedBuffer(data); + } + } + + @Test + public void test() + { + BlockData data = new BlockData(); + data.setup(); + benchmarkAppendRange(data); + } + + public static void main(String[] args) + throws Exception + { + benchmark(BenchmarkBlockBuilder.class).run(); + } +} diff --git a/core/trino-main/src/test/java/io/trino/block/BenchmarkMapCopy.java b/core/trino-main/src/test/java/io/trino/block/BenchmarkMapCopy.java index 9fb639b32cfc..38c95ecaecf9 100644 --- a/core/trino-main/src/test/java/io/trino/block/BenchmarkMapCopy.java +++ b/core/trino-main/src/test/java/io/trino/block/BenchmarkMapCopy.java @@ -55,10 +55,9 @@ public BlockBuilder benchmarkMapCopy(BenchmarkData data) { Block block = data.getDataBlock(); BlockBuilder blockBuilder = data.getBlockBuilder(); - MapType mapType = mapType(VARCHAR, BIGINT); for (int i = 0; i < POSITIONS; i++) { - mapType.appendTo(block, i, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(i)); } return blockBuilder; diff --git a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java index 2aa537683338..bd6a743009b6 100644 --- a/core/trino-main/src/test/java/io/trino/connector/MockConnector.java +++ b/core/trino-main/src/test/java/io/trino/connector/MockConnector.java @@ -118,6 +118,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -143,7 +144,9 @@ public class MockConnector private static final String UPDATE_ROW_ID = "update_row_id"; private static final String MERGE_ROW_ID = "merge_row_id"; + private final List> sessionProperties; private final Function metadataWrapper; + private final Consumer cleanupQuery; private final Function> listSchemaNames; private final BiFunction> listTables; private final Optional>> streamTableColumns; @@ -188,7 +191,6 @@ public class MockConnector private final Supplier>> schemaProperties; private final Supplier>> tableProperties; private final Supplier>> columnProperties; - private final List> sessionProperties; private final Function tableFunctionSplitsSources; private final OptionalInt maxWriterTasks; private final BiFunction> getLayoutForTableExecute; @@ -197,8 +199,9 @@ public class MockConnector private final boolean allowSplittingReadIntoMultipleSubQueries; MockConnector( - Function metadataWrapper, List> sessionProperties, + Function metadataWrapper, + Consumer cleanupQuery, Function> listSchemaNames, BiFunction> listTables, Optional>> streamTableColumns, @@ -250,8 +253,9 @@ public class MockConnector Supplier> capabilities, boolean allowSplittingReadIntoMultipleSubQueries) { - this.metadataWrapper = requireNonNull(metadataWrapper, "metadataWrapper is null"); this.sessionProperties = ImmutableList.copyOf(requireNonNull(sessionProperties, "sessionProperties is null")); + this.metadataWrapper = requireNonNull(metadataWrapper, "metadataWrapper is null"); + this.cleanupQuery = requireNonNull(cleanupQuery, "cleanupQuery is null"); this.listSchemaNames = requireNonNull(listSchemaNames, "listSchemaNames is null"); this.listTables = requireNonNull(listTables, "listTables is null"); this.streamTableColumns = requireNonNull(streamTableColumns, "streamTableColumns is null"); @@ -882,7 +886,10 @@ public Optional getTableHandleForExecute(ConnectorS } @Override - public void executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) {} + public Map executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) + { + return ImmutableMap.of(); + } @Override public void finishTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle, Collection fragments, List tableExecuteState) {} @@ -1014,6 +1021,12 @@ private MockConnectorAccessControl getMockAccessControl() { return (MockConnectorAccessControl) getAccessControl(); } + + @Override + public void cleanupQuery(ConnectorSession session) + { + cleanupQuery.accept(session); + } } private static class MockPageSinkProvider diff --git a/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java b/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java index 1fc09c5756c3..e5edbc1d059d 100644 --- a/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java +++ b/core/trino-main/src/test/java/io/trino/connector/MockConnectorFactory.java @@ -78,6 +78,7 @@ import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.BiFunction; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -94,8 +95,9 @@ public class MockConnectorFactory implements ConnectorFactory { private final String name; - private final List> sessionProperty; + private final List> sessionProperties; private final Function metadataWrapper; + private final Consumer cleanupQuery; private final Function> listSchemaNames; private final BiFunction> listTables; private final Optional>> streamTableColumns; @@ -152,8 +154,9 @@ public class MockConnectorFactory private MockConnectorFactory( String name, - List> sessionProperty, + List> sessionProperties, Function metadataWrapper, + Consumer cleanupQuery, Function> listSchemaNames, BiFunction> listTables, Optional>> streamTableColumns, @@ -206,8 +209,9 @@ private MockConnectorFactory( boolean allowSplittingReadIntoMultipleSubQueries) { this.name = requireNonNull(name, "name is null"); - this.sessionProperty = ImmutableList.copyOf(requireNonNull(sessionProperty, "sessionProperty is null")); + this.sessionProperties = ImmutableList.copyOf(requireNonNull(sessionProperties, "sessionProperties is null")); this.metadataWrapper = requireNonNull(metadataWrapper, "metadataWrapper is null"); + this.cleanupQuery = requireNonNull(cleanupQuery, "cleanupQuery is null"); this.listSchemaNames = requireNonNull(listSchemaNames, "listSchemaNames is null"); this.listTables = requireNonNull(listTables, "listTables is null"); this.streamTableColumns = requireNonNull(streamTableColumns, "streamTableColumns is null"); @@ -270,8 +274,9 @@ public String getName() public Connector create(String catalogName, Map config, ConnectorContext context) { return new MockConnector( + sessionProperties, metadataWrapper, - sessionProperty, + cleanupQuery, listSchemaNames, listTables, streamTableColumns, @@ -421,6 +426,7 @@ public static final class Builder { private String name = "mock"; private final List> sessionProperties = new ArrayList<>(); + private Consumer cleanupQuery = session -> {}; private Function metadataWrapper = identity(); private Function> listSchemaNames = defaultListSchemaNames(); private BiFunction> listTables = defaultListTables(); @@ -501,6 +507,12 @@ public Builder withSessionProperties(Iterable> sessionProper return this; } + public Builder withCleanupQuery(Consumer cleanupQuery) + { + this.cleanupQuery = cleanupQuery; + return this; + } + public Builder withMetadataWrapper(Function metadataWrapper) { this.metadataWrapper = requireNonNull(metadataWrapper, "metadataWrapper is null"); @@ -843,6 +855,7 @@ public MockConnectorFactory build() name, sessionProperties, metadataWrapper, + cleanupQuery, listSchemaNames, listTables, streamTableColumns, diff --git a/core/trino-main/src/test/java/io/trino/cost/TestAggregationStatsRule.java b/core/trino-main/src/test/java/io/trino/cost/TestAggregationStatsRule.java index 016640833702..8e9276f58a55 100644 --- a/core/trino-main/src/test/java/io/trino/cost/TestAggregationStatsRule.java +++ b/core/trino-main/src/test/java/io/trino/cost/TestAggregationStatsRule.java @@ -176,7 +176,7 @@ public void testAggregationWithMoreGroupingSets() .aggregation(ab -> ab .addAggregation(pb.symbol("count_on_x", BIGINT), aggregation("count", ImmutableList.of(new Reference(BIGINT, "x"))), ImmutableList.of(BIGINT)) .addAggregation(pb.symbol("sum", BIGINT), aggregation("sum", ImmutableList.of(new Reference(BIGINT, "x"))), ImmutableList.of(BIGINT)) - .groupingSets(new AggregationNode.GroupingSetDescriptor(ImmutableList.of(pb.symbol("y"), pb.symbol("z")), 3, ImmutableSet.of(0))) + .groupingSets(new AggregationNode.GroupingSetDescriptor(ImmutableSet.of(pb.symbol("y"), pb.symbol("z")), 3, ImmutableSet.of(0))) .source(pb.values(pb.symbol("x", BIGINT), pb.symbol("y", BIGINT), pb.symbol("z", BIGINT))))) .withSourceStats(PlanNodeStatsEstimate.builder() .setOutputRowCount(100) diff --git a/core/trino-main/src/test/java/io/trino/execution/BaseTestSqlTaskManager.java b/core/trino-main/src/test/java/io/trino/execution/BaseTestSqlTaskManager.java index 6326556b3519..ae128f8d1b13 100644 --- a/core/trino-main/src/test/java/io/trino/execution/BaseTestSqlTaskManager.java +++ b/core/trino-main/src/test/java/io/trino/execution/BaseTestSqlTaskManager.java @@ -272,8 +272,8 @@ public void testSessionPropertyMemoryLimitOverride() TaskId reduceLimitsId = new TaskId(new StageId("q1", 0), 1, 0); TaskId increaseLimitsId = new TaskId(new StageId("q2", 0), 1, 0); - QueryContext reducesLimitsContext = sqlTaskManager.getQueryContext(reduceLimitsId.getQueryId()); - QueryContext attemptsIncreaseContext = sqlTaskManager.getQueryContext(increaseLimitsId.getQueryId()); + QueryContext reducesLimitsContext = sqlTaskManager.getQueryContext(reduceLimitsId.queryId()); + QueryContext attemptsIncreaseContext = sqlTaskManager.getQueryContext(increaseLimitsId.queryId()); // not initialized with a task update yet assertThat(reducesLimitsContext.isMemoryLimitsInitialized()).isFalse(); @@ -354,7 +354,7 @@ private TaskInfo createTask(SqlTaskManager sqlTaskManager, TaskId taskId, Set {}, false, false); return sqlTaskManager.updateTask(TEST_SESSION, taskId, diff --git a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java index d385309479a5..cf25e47851f6 100644 --- a/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java +++ b/core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java @@ -266,6 +266,7 @@ public QueryInfo getFullQueryInfo() ImmutableList.of(), DynamicFiltersStats.EMPTY, + ImmutableMap.of(), ImmutableList.of(), ImmutableList.of()), Optional.empty(), diff --git a/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java b/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java index 017cccf5bbc6..8f73d91515c3 100644 --- a/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java +++ b/core/trino-main/src/test/java/io/trino/execution/MockRemoteTaskFactory.java @@ -214,7 +214,7 @@ public MockRemoteTask( MemoryPool memoryPool = new MemoryPool(DataSize.of(1, GIGABYTE)); SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE)); - QueryContext queryContext = new QueryContext(taskId.getQueryId(), + QueryContext queryContext = new QueryContext(taskId.queryId(), DataSize.of(1, MEGABYTE), memoryPool, new TestingGcMonitor(), diff --git a/core/trino-main/src/test/java/io/trino/execution/TaskTestUtils.java b/core/trino-main/src/test/java/io/trino/execution/TaskTestUtils.java index 481f408c4bf3..618453c7a53f 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TaskTestUtils.java +++ b/core/trino-main/src/test/java/io/trino/execution/TaskTestUtils.java @@ -161,12 +161,12 @@ public static LocalExecutionPlanner createTestingPlanner() new JoinFilterFunctionCompiler(PLANNER_CONTEXT.getFunctionManager()), new IndexJoinLookupStats(), new TaskManagerConfig(), - new GenericSpillerFactory((types, spillContext, memoryContext) -> { + new GenericSpillerFactory((types, spillContext, memoryContext, parallelSpill) -> { throw new UnsupportedOperationException(); }), new QueryDataEncoders(new SpoolingEnabledConfig(), Set.of()), Optional.empty(), - (types, spillContext, memoryContext) -> { + (types, spillContext, memoryContext, parallelSpill) -> { throw new UnsupportedOperationException(); }, (types, partitionFunction, spillContext, memoryContext) -> { diff --git a/core/trino-main/src/test/java/io/trino/execution/TestDistributionSnapshot.java b/core/trino-main/src/test/java/io/trino/execution/TestDistributionSnapshot.java index e96d85a03076..579649e11a68 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestDistributionSnapshot.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestDistributionSnapshot.java @@ -14,6 +14,7 @@ package io.trino.execution; import io.airlift.stats.TDigest; +import io.trino.plugin.base.metrics.DistributionSnapshot; import io.trino.plugin.base.metrics.TDigestHistogram; import org.junit.jupiter.api.Test; @@ -36,7 +37,7 @@ void testConvertFromTDigestHistogram() digest.add(7.0); TDigestHistogram histogram = new TDigestHistogram(digest); - DistributionSnapshot snapshot = new DistributionSnapshot(histogram); + DistributionSnapshot snapshot = DistributionSnapshot.fromDistribution(histogram); assertThat(snapshot.total()).isEqualTo(9); assertThat(snapshot.min()).isEqualTo(1.0); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestDropCatalogTask.java b/core/trino-main/src/test/java/io/trino/execution/TestDropCatalogTask.java index dea0fc271724..42358b3bcf81 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestDropCatalogTask.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestDropCatalogTask.java @@ -90,7 +90,7 @@ public void testDuplicatedCreateCatalog() } @Test - public void testDuplicatedCreateCatalogIfNotExists() + public void testDuplicatedDropCatalogIfNotExists() { queryRunner.createCatalog(TEST_CATALOG, "tpch", ImmutableMap.of()); assertThat(queryRunner.getPlannerContext().getMetadata().catalogExists(createNewQuery().getSession(), TEST_CATALOG)).isTrue(); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java b/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java index 883f337e857a..f6213664e96b 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestMemoryRevokingScheduler.java @@ -268,7 +268,7 @@ private SqlTask newSqlTask(QueryId queryId) { QueryContext queryContext = getOrCreateQueryContext(queryId); - TaskId taskId = new TaskId(new StageId(queryId.getId(), 0), idGenerator.incrementAndGet(), 0); + TaskId taskId = new TaskId(new StageId(queryId.id(), 0), idGenerator.incrementAndGet(), 0); URI location = URI.create("fake://task/" + taskId); return createSqlTask( diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryInfo.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryInfo.java index 79d2c17f943d..7558f25e5f39 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryInfo.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryInfo.java @@ -28,6 +28,7 @@ import io.opentelemetry.api.trace.Span; import io.trino.client.NodeVersion; import io.trino.operator.RetryPolicy; +import io.trino.plugin.base.metrics.DistributionSnapshot; import io.trino.plugin.base.metrics.LongCount; import io.trino.plugin.base.metrics.TDigestHistogram; import io.trino.server.BasicQueryStats; @@ -318,7 +319,7 @@ private static StageStats createStageStats(int value) Duration.succinctDuration(value, SECONDS), Duration.succinctDuration(value, SECONDS), succinctBytes(value), - Optional.of(new DistributionSnapshot(new TDigestHistogram(new TDigest()))), + Optional.of(DistributionSnapshot.fromDistribution(new TDigestHistogram(new TDigest()))), succinctBytes(value), succinctBytes(value), value, diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java index f6e428917b76..f75b569e3e57 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java @@ -17,13 +17,16 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.trino.operator.RetryPolicy; +import jakarta.validation.constraints.AssertTrue; import org.junit.jupiter.api.Test; +import java.util.EnumSet; import java.util.Map; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; import static io.airlift.units.DataSize.Unit.GIGABYTE; import static io.airlift.units.DataSize.Unit.KILOBYTE; import static io.airlift.units.DataSize.Unit.MEGABYTE; @@ -76,6 +79,7 @@ public void testDefaults() .setRequiredWorkers(1) .setRequiredWorkersMaxWait(new Duration(5, MINUTES)) .setRetryPolicy(RetryPolicy.NONE) + .setAllowedRetryPolicies(EnumSet.allOf(RetryPolicy.class)) .setQueryRetryAttempts(4) .setTaskRetryAttemptsPerTask(4) .setRetryInitialDelay(new Duration(10, SECONDS)) @@ -160,6 +164,7 @@ public void testExplicitPropertyMappings() .put("query-manager.required-workers", "333") .put("query-manager.required-workers-max-wait", "33m") .put("retry-policy", "QUERY") + .put("retry-policy.allowed", "QUERY,TASK") .put("query-retry-attempts", "0") .put("task-retry-attempts-per-task", "9") .put("retry-initial-delay", "1m") @@ -241,6 +246,7 @@ public void testExplicitPropertyMappings() .setRequiredWorkers(333) .setRequiredWorkersMaxWait(new Duration(33, MINUTES)) .setRetryPolicy(RetryPolicy.QUERY) + .setAllowedRetryPolicies(EnumSet.of(RetryPolicy.QUERY, RetryPolicy.TASK)) .setQueryRetryAttempts(0) .setTaskRetryAttemptsPerTask(9) .setRetryInitialDelay(new Duration(1, MINUTES)) @@ -290,4 +296,16 @@ public void testExplicitPropertyMappings() assertFullMapping(properties, expected); } + + @Test + public void testAllowedRetryPoliciesValidation() + { + assertFailsValidation( + new QueryManagerConfig() + .setAllowedRetryPolicies(EnumSet.of(RetryPolicy.NONE, RetryPolicy.TASK)) + .setRetryPolicy(RetryPolicy.QUERY), + "retryPolicyAllowed", + "Selected retry policy not present in retry-policy.allowed list", + AssertTrue.class); + } } diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryStateMachine.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryStateMachine.java index 7392fa0a2a5b..9b11c043def4 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryStateMachine.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryStateMachine.java @@ -30,6 +30,7 @@ import io.trino.execution.warnings.WarningCollector; import io.trino.execution.warnings.WarningCollectorConfig; import io.trino.metadata.Metadata; +import io.trino.metadata.TestMetadataManager; import io.trino.plugin.base.security.AllowAllSystemAccessControl; import io.trino.plugin.base.security.DefaultSystemAccessControl; import io.trino.security.AccessControlConfig; @@ -89,7 +90,6 @@ import static io.trino.execution.QueryState.STARTING; import static io.trino.execution.QueryState.WAITING_FOR_RESOURCES; import static io.trino.execution.querystats.PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector; -import static io.trino.metadata.TestMetadataManager.createTestMetadataManager; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static io.trino.spi.StandardErrorCode.TYPE_MISMATCH; import static io.trino.spi.StandardErrorCode.USER_CANCELED; @@ -442,15 +442,7 @@ public void testPreserveFirstFailure() { CountDownLatch cleanup = new CountDownLatch(1); QueryStateMachine queryStateMachine = queryStateMachine() - .withMetadata(new TracingMetadata(noopTracer(), createTestMetadataManager()) - { - @Override - public void cleanupQuery(Session session) - { - cleanup.countDown(); - super.cleanupQuery(session); - } - }) + .beforeQueryCleanup(cleanup::countDown) .build(); Future anotherThread = executor.submit(() -> { @@ -479,15 +471,7 @@ public void testPreserveCancellation() { CountDownLatch cleanup = new CountDownLatch(1); QueryStateMachine queryStateMachine = queryStateMachine() - .withMetadata(new TracingMetadata(noopTracer(), createTestMetadataManager()) - { - @Override - public void cleanupQuery(Session session) - { - cleanup.countDown(); - super.cleanupQuery(session); - } - }) + .beforeQueryCleanup(cleanup::countDown) .build(); Future anotherThread = executor.submit(() -> { @@ -761,7 +745,7 @@ private QueryStateMachineBuilder queryStateMachine() private class QueryStateMachineBuilder { private Ticker ticker = Ticker.systemTicker(); - private Metadata metadata; + private Optional beforeQueryCleanup = Optional.empty(); private WarningCollector warningCollector = WarningCollector.NOOP; private String setCatalog; private String setPath; @@ -780,9 +764,9 @@ public QueryStateMachineBuilder withTicker(Ticker ticker) } @CanIgnoreReturnValue - public QueryStateMachineBuilder withMetadata(Metadata metadata) + public QueryStateMachineBuilder beforeQueryCleanup(Runnable runnable) { - this.metadata = metadata; + this.beforeQueryCleanup = Optional.of(runnable); return this; } @@ -842,10 +826,24 @@ public QueryStateMachineBuilder withAddPreparedStatements(Map pr public QueryStateMachine build() { - if (metadata == null) { - metadata = createTestMetadataManager(); - } TransactionManager transactionManager = createTestTransactionManager(); + Metadata metadata = TestMetadataManager.builder() + .withTransactionManager(transactionManager) + .build(); + if (beforeQueryCleanup.isPresent()) { + Runnable beforeQueryCleanupAction = beforeQueryCleanup.get(); + // Using TracingMetadata in lieu of "Forwarding Metadata" which currently does not exist + metadata = new TracingMetadata(noopTracer(), metadata) + { + @Override + public void cleanupQuery(Session session) + { + beforeQueryCleanupAction.run(); + super.cleanupQuery(session); + } + }; + } + AccessControlManager accessControl = new AccessControlManager( NodeVersion.UNKNOWN, transactionManager, diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java index ed2587797ffb..7dd7d1c6b479 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java @@ -14,6 +14,7 @@ package io.trino.execution; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.airlift.json.JsonCodec; import io.airlift.units.DataSize; @@ -270,7 +271,7 @@ public class TestQueryStats 107)), DynamicFiltersStats.EMPTY, - + ImmutableMap.of(), operatorSummaries, optimizerRulesSummaries); diff --git a/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java b/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java index 85c2999db29b..b31ad64e2b39 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestStageStats.java @@ -92,7 +92,7 @@ public class TestStageStats new Duration(202, NANOSECONDS), DataSize.ofBytes(34), - Optional.of(new io.trino.execution.DistributionSnapshot(getTDigestHistogram(10))), + Optional.of(io.trino.plugin.base.metrics.DistributionSnapshot.fromDistribution(getTDigestHistogram(10))), DataSize.ofBytes(35), DataSize.ofBytes(36), 37, diff --git a/core/trino-main/src/test/java/io/trino/execution/scheduler/TestScaledWriterScheduler.java b/core/trino-main/src/test/java/io/trino/execution/scheduler/TestScaledWriterScheduler.java index 0ca4ce417c79..baaef875acfe 100644 --- a/core/trino-main/src/test/java/io/trino/execution/scheduler/TestScaledWriterScheduler.java +++ b/core/trino-main/src/test/java/io/trino/execution/scheduler/TestScaledWriterScheduler.java @@ -34,6 +34,7 @@ import io.trino.metadata.Split; import io.trino.node.InternalNode; import io.trino.node.TestingInternalNodeManager; +import io.trino.spi.QueryId; import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.sql.planner.Partitioning; @@ -229,7 +230,7 @@ private static TaskStatus buildTaskStatus(int maxWriterCount, DataSize writerInp private static TaskStatus buildTaskStatus(boolean isOutputBufferOverUtilized, long outputDataSize, Optional maxWriterCount, DataSize writerInputDataSize) { return new TaskStatus( - TaskId.valueOf("taskId"), + new TaskId(new StageId(new QueryId("query_id"), 0), 0, 0), "task-instance-id", 0, TaskState.RUNNING, @@ -258,10 +259,12 @@ private static class TestingStageExecution implements StageExecution { private final PlanFragment fragment; + private final StageId stageId; public TestingStageExecution(PlanFragment fragment) { this.fragment = requireNonNull(fragment, "fragment is null"); + this.stageId = new StageId(new QueryId("query_id"), 0); } @Override @@ -357,7 +360,7 @@ public void recordSplitSourceMetrics(PlanNodeId nodeId, Metrics metrics, long st @Override public Optional scheduleTask(InternalNode node, int partition, Multimap initialSplits) { - return Optional.of(new TestingRemoteTask(TaskId.valueOf("taskId"), "nodeId", fragment)); + return Optional.of(new TestingRemoteTask(new TaskId(stageId, partition, 0), "nodeId", fragment)); } @Override diff --git a/core/trino-main/src/test/java/io/trino/execution/scheduler/TestSchedulingUtils.java b/core/trino-main/src/test/java/io/trino/execution/scheduler/TestSchedulingUtils.java index baeeb450f88d..9f93ae3a4091 100644 --- a/core/trino-main/src/test/java/io/trino/execution/scheduler/TestSchedulingUtils.java +++ b/core/trino-main/src/test/java/io/trino/execution/scheduler/TestSchedulingUtils.java @@ -72,6 +72,34 @@ public void testCanStreamNoJoin() assertThat(SchedulingUtils.canStream(parentSubPlan, valuesSubPlan("a"))).isTrue(); } + @Test + public void testCanStreamUnion() + { + /* + parent(union) + / \ \ + -------------------------------------- stage boundary + a b c + */ + SubPlan aSubPlan = valuesSubPlan("a"); + RemoteSourceNode remoteSourceA = remoteSource("a"); + + SubPlan bSubPlan = valuesSubPlan("b"); + RemoteSourceNode remoteSourceB = remoteSource("b"); + + SubPlan cSubPlan = valuesSubPlan("c"); + RemoteSourceNode remoteSourceC = remoteSource("c"); + + SubPlan parentSubPlan = createSubPlan( + "parent", + union("union", ImmutableList.of(remoteSourceA, remoteSourceB, remoteSourceC)), + ImmutableList.of(aSubPlan, bSubPlan, cSubPlan)); + + assertThat(SchedulingUtils.canStream(parentSubPlan, aSubPlan)).isTrue(); + assertThat(SchedulingUtils.canStream(parentSubPlan, bSubPlan)).isTrue(); + assertThat(SchedulingUtils.canStream(parentSubPlan, cSubPlan)).isTrue(); + } + @Test public void testCanStreamJoin() { diff --git a/core/trino-main/src/test/java/io/trino/execution/scheduler/faulttolerant/BenchmarkBinPackingNodeAllocator.java b/core/trino-main/src/test/java/io/trino/execution/scheduler/faulttolerant/BenchmarkBinPackingNodeAllocator.java index 3b5075e47992..b97743a39175 100644 --- a/core/trino-main/src/test/java/io/trino/execution/scheduler/faulttolerant/BenchmarkBinPackingNodeAllocator.java +++ b/core/trino-main/src/test/java/io/trino/execution/scheduler/faulttolerant/BenchmarkBinPackingNodeAllocator.java @@ -179,6 +179,7 @@ private MemoryInfo buildWorkerMemoryInfo(DataSize usedMemory, Map toNodeMemoryInfoList(long memoryPoolMaxBytes, Map executeTableExecute(Session session, TableExecuteHandle handle) { throw new UnsupportedOperationException(); } @@ -205,6 +208,12 @@ public Optional getInfo(Session session, TableHandle handle) throw new UnsupportedOperationException(); } + @Override + public Metrics getMetrics(Session session, String catalogName) + { + throw new UnsupportedOperationException(); + } + @Override public CatalogSchemaTableName getTableName(Session session, TableHandle tableHandle) { @@ -265,6 +274,18 @@ public List listRelationComments(Session session, Strin throw new UnsupportedOperationException(); } + @Override + public void createCatalog(Session session, CatalogName catalog, ConnectorName connectorName, Map properties, boolean notExists) + { + throw new UnsupportedOperationException(); + } + + @Override + public void dropCatalog(Session session, CatalogName catalog, boolean cascade) + { + throw new UnsupportedOperationException(); + } + @Override public void createSchema(Session session, CatalogSchemaName schema, Map properties, TrinoPrincipal principal) { @@ -422,7 +443,7 @@ public Optional getInsertLayout(Session session, TableHandle target } @Override - public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata) + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Session session, CatalogHandle catalogHandle, ConnectorTableMetadata tableMetadata, boolean tableReplace) { throw new UnsupportedOperationException(); } @@ -566,6 +587,12 @@ public List listCatalogs(Session session) throw new UnsupportedOperationException(); } + @Override + public List listActiveCatalogs(Session session) + { + throw new UnsupportedOperationException(); + } + @Override public List listViews(Session session, QualifiedTablePrefix prefix) { diff --git a/core/trino-main/src/test/java/io/trino/metadata/TestMetadataManager.java b/core/trino-main/src/test/java/io/trino/metadata/TestMetadataManager.java index fc3d922260ef..5e1c2f09a304 100644 --- a/core/trino-main/src/test/java/io/trino/metadata/TestMetadataManager.java +++ b/core/trino-main/src/test/java/io/trino/metadata/TestMetadataManager.java @@ -28,6 +28,7 @@ import java.util.Set; import static io.trino.client.NodeVersion.UNKNOWN; +import static io.trino.metadata.CatalogManager.NO_CATALOGS; import static io.trino.transaction.InMemoryTransactionManager.createTestTransactionManager; import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; import static java.util.Objects.requireNonNull; @@ -111,7 +112,8 @@ public MetadataManager build() globalFunctionCatalog, languageFunctionManager, tableFunctionRegistry, - typeManager); + typeManager, + NO_CATALOGS); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/BenchmarkGroupByHashOnSimulatedData.java b/core/trino-main/src/test/java/io/trino/operator/BenchmarkGroupByHashOnSimulatedData.java index 02e6a8fca299..d04391f05933 100644 --- a/core/trino-main/src/test/java/io/trino/operator/BenchmarkGroupByHashOnSimulatedData.java +++ b/core/trino-main/src/test/java/io/trino/operator/BenchmarkGroupByHashOnSimulatedData.java @@ -560,7 +560,7 @@ private void createNonDictionaryBlock(int blockCount, int positionsPerBlock, int } else { int position = r.nextInt(distinctValuesCountInColumn); - columnType.getType().appendTo(allValues, position, block); + block.append(allValues.getUnderlyingValueBlock(), allValues.getUnderlyingValuePosition(position)); } } blocks[i] = block.build(); diff --git a/core/trino-main/src/test/java/io/trino/operator/BenchmarkPagesIndexOrdering.java b/core/trino-main/src/test/java/io/trino/operator/BenchmarkPagesIndexOrdering.java index 0c9b49407d68..af9faa0a89b0 100644 --- a/core/trino-main/src/test/java/io/trino/operator/BenchmarkPagesIndexOrdering.java +++ b/core/trino-main/src/test/java/io/trino/operator/BenchmarkPagesIndexOrdering.java @@ -13,10 +13,13 @@ */ package io.trino.operator; +import io.airlift.slice.Slices; import io.trino.RowPagesBuilder; import io.trino.spi.Page; import io.trino.spi.connector.SortOrder; +import io.trino.spi.type.BigintType; import io.trino.spi.type.Type; +import io.trino.spi.type.VarcharType; import org.junit.jupiter.api.Test; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -39,6 +42,7 @@ import static io.trino.jmh.Benchmarks.benchmark; import static io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST; import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.VARCHAR; import static java.util.Collections.nCopies; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; @@ -49,8 +53,8 @@ @OutputTimeUnit(MILLISECONDS) @BenchmarkMode(AverageTime) @Fork(2) -@Warmup(iterations = 3) -@Measurement(iterations = 10, time = 2, timeUnit = SECONDS) +@Warmup(iterations = 10, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 10, time = 1, timeUnit = SECONDS) public class BenchmarkPagesIndexOrdering { private static final Random RANDOM = new Random(633969769); @@ -62,6 +66,8 @@ public static class Context { @Param({"1", "10"}) protected int numberOfChannels = 1; + @Param({"BIGINT", "VARCHAR"}) + protected String typeName = "BIGINT"; private List types; private List sortChannels; @@ -71,7 +77,8 @@ public static class Context @Setup public void setup() { - types = nCopies(numberOfChannels, BIGINT); + Type type = getType(typeName); + types = nCopies(numberOfChannels, type); sortChannels = IntStream.range(0, numberOfChannels).boxed().collect(toImmutableList()); sortOrders = nCopies(numberOfChannels, ASC_NULLS_FIRST); @@ -80,7 +87,11 @@ public void setup() for (int row = 0; row < ROWS_PER_PAGE; row++) { Object[] values = new Object[numberOfChannels]; for (int channel = 0; channel < numberOfChannels; channel++) { - values[channel] = RANDOM.nextLong(); + values[channel] = switch (type) { + case BigintType _ -> RANDOM.nextLong(); + case VarcharType _ -> Slices.utf8Slice(String.valueOf(RANDOM.nextLong())); + default -> throw new IllegalArgumentException("Unsupported type: " + typeName); + }; } pagesBuilder.row(values); } @@ -88,6 +99,15 @@ public void setup() } pages = pagesBuilder.build(); } + + private static Type getType(String typeName) + { + return switch (typeName) { + case "BIGINT" -> BIGINT; + case "VARCHAR" -> VARCHAR; + default -> throw new IllegalArgumentException("Unsupported type: " + typeName); + }; + } } @Benchmark diff --git a/core/trino-main/src/test/java/io/trino/operator/aggregation/AbstractTestAggregationFunction.java b/core/trino-main/src/test/java/io/trino/operator/aggregation/AbstractTestAggregationFunction.java index 7198a78ed8e3..36693de870ba 100644 --- a/core/trino-main/src/test/java/io/trino/operator/aggregation/AbstractTestAggregationFunction.java +++ b/core/trino-main/src/test/java/io/trino/operator/aggregation/AbstractTestAggregationFunction.java @@ -198,14 +198,15 @@ protected static Block[] createAlternatingNullsBlock(List types, Block... { Block[] alternatingNullsBlocks = new Block[sequenceBlocks.length]; for (int i = 0; i < sequenceBlocks.length; i++) { - int positionCount = sequenceBlocks[i].getPositionCount(); + Block block = sequenceBlocks[i]; + int positionCount = block.getPositionCount(); Type type = types.get(i); BlockBuilder blockBuilder = type.createBlockBuilder(null, positionCount); for (int position = 0; position < positionCount; position++) { // append null blockBuilder.appendNull(); // append value - type.appendTo(sequenceBlocks[i], position, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } alternatingNullsBlocks[i] = blockBuilder.build(); } diff --git a/core/trino-main/src/test/java/io/trino/operator/join/BenchmarkHashBuildAndJoinOperators.java b/core/trino-main/src/test/java/io/trino/operator/join/BenchmarkHashBuildAndJoinOperators.java index b2e07358939f..293d464d55ee 100644 --- a/core/trino-main/src/test/java/io/trino/operator/join/BenchmarkHashBuildAndJoinOperators.java +++ b/core/trino-main/src/test/java/io/trino/operator/join/BenchmarkHashBuildAndJoinOperators.java @@ -29,6 +29,7 @@ import io.trino.operator.join.HashBuilderOperator.HashBuilderOperatorFactory; import io.trino.spi.Page; import io.trino.spi.PageBuilder; +import io.trino.spi.block.Block; import io.trino.spi.type.Type; import io.trino.spi.type.TypeOperators; import io.trino.spiller.SingleStreamSpillerFactory; @@ -401,8 +402,8 @@ private static void appendRow(PageBuilder pageBuilder, List types, Page pa pageBuilder.declarePosition(); for (int channel = 0; channel < types.size(); channel++) { - Type type = types.get(channel); - type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel)); + Block block = page.getBlock(channel); + pageBuilder.getBlockBuilder(channel).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/join/JoinTestUtils.java b/core/trino-main/src/test/java/io/trino/operator/join/JoinTestUtils.java index c24c2b8a7a59..5290d731b238 100644 --- a/core/trino-main/src/test/java/io/trino/operator/join/JoinTestUtils.java +++ b/core/trino-main/src/test/java/io/trino/operator/join/JoinTestUtils.java @@ -318,7 +318,7 @@ public DummySpillerFactory failUnspill() } @Override - public SingleStreamSpiller create(List types, SpillContext spillContext, LocalMemoryContext memoryContext) + public SingleStreamSpiller create(List types, SpillContext spillContext, LocalMemoryContext memoryContext, boolean parallelSpill) { return new SingleStreamSpiller() { diff --git a/core/trino-main/src/test/java/io/trino/operator/join/TestLookupJoinPageBuilder.java b/core/trino-main/src/test/java/io/trino/operator/join/TestLookupJoinPageBuilder.java index 6616b7aacaa1..340c6ac6e1aa 100644 --- a/core/trino-main/src/test/java/io/trino/operator/join/TestLookupJoinPageBuilder.java +++ b/core/trino-main/src/test/java/io/trino/operator/join/TestLookupJoinPageBuilder.java @@ -237,7 +237,8 @@ public boolean isJoinPositionEligible(long currentJoinPosition, int probePositio public void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset) { for (int i = 0; i < types.size(); i++) { - types.get(i).appendTo(page.getBlock(i), (int) position, pageBuilder.getBlockBuilder(i)); + Block block = page.getBlock(i); + pageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition((int) position)); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/join/unspilled/BenchmarkHashBuildAndJoinOperators.java b/core/trino-main/src/test/java/io/trino/operator/join/unspilled/BenchmarkHashBuildAndJoinOperators.java index 5f3be9dfb0b1..44a6b905ba43 100644 --- a/core/trino-main/src/test/java/io/trino/operator/join/unspilled/BenchmarkHashBuildAndJoinOperators.java +++ b/core/trino-main/src/test/java/io/trino/operator/join/unspilled/BenchmarkHashBuildAndJoinOperators.java @@ -31,6 +31,7 @@ import io.trino.operator.join.unspilled.HashBuilderOperator.HashBuilderOperatorFactory; import io.trino.spi.Page; import io.trino.spi.PageBuilder; +import io.trino.spi.block.Block; import io.trino.spi.type.Type; import io.trino.spi.type.TypeOperators; import io.trino.sql.planner.plan.PlanNodeId; @@ -399,8 +400,8 @@ private static void appendRow(PageBuilder pageBuilder, List types, Page pa pageBuilder.declarePosition(); for (int channel = 0; channel < types.size(); channel++) { - Type type = types.get(channel); - type.appendTo(page.getBlock(channel), position, pageBuilder.getBlockBuilder(channel)); + Block block = page.getBlock(channel); + pageBuilder.getBlockBuilder(channel).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/join/unspilled/TestLookupJoinPageBuilder.java b/core/trino-main/src/test/java/io/trino/operator/join/unspilled/TestLookupJoinPageBuilder.java index ecd2518b1a8d..d659f73b1db8 100644 --- a/core/trino-main/src/test/java/io/trino/operator/join/unspilled/TestLookupJoinPageBuilder.java +++ b/core/trino-main/src/test/java/io/trino/operator/join/unspilled/TestLookupJoinPageBuilder.java @@ -238,7 +238,8 @@ public boolean isJoinPositionEligible(long currentJoinPosition, int probePositio public void appendTo(long position, PageBuilder pageBuilder, int outputChannelOffset) { for (int i = 0; i < types.size(); i++) { - types.get(i).appendTo(page.getBlock(i), (int) position, pageBuilder.getBlockBuilder(i)); + Block block = page.getBlock(i); + pageBuilder.getBlockBuilder(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition((int) position)); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/output/TestPagePartitioner.java b/core/trino-main/src/test/java/io/trino/operator/output/TestPagePartitioner.java index 4318256ad669..a7b8c81589ce 100644 --- a/core/trino-main/src/test/java/io/trino/operator/output/TestPagePartitioner.java +++ b/core/trino-main/src/test/java/io/trino/operator/output/TestPagePartitioner.java @@ -126,14 +126,23 @@ public void tearDownClass() public void testOutputForEmptyPage() { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).build(); Page page = new Page(createLongsBlock(ImmutableList.of())); - pagePartitioner.partitionPage(page, operatorContext()); - pagePartitioner.close(); + multiPartitionPartitioner.partitionPage(page, operatorContext()); + multiPartitionPartitioner.close(); List partitioned = readLongs(outputBuffer.getEnqueuedDeserialized(), 0); assertThat(partitioned).isEmpty(); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionResult = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionResult).isEmpty(); } private OperatorContext operatorContext() @@ -153,14 +162,23 @@ private void testOutputEqualsInput(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).build(); Page page = new Page(createLongSequenceBlock(0, POSITIONS_PER_PAGE)); List expected = readLongs(Stream.of(page), 0); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partitioned = readLongs(outputBuffer.getEnqueuedDeserialized(), 0); assertThat(partitioned).containsExactlyInAnyOrderElementsOf(expected); // order is different due to 2 partitions joined + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionResult = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionResult).containsExactlyElementsOf(expected); } @Test @@ -201,16 +219,28 @@ private void testOutputForMultipleSimplePages(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).build(); Page page1 = new Page(createLongSequenceBlock(0, POSITIONS_PER_PAGE)); Page page2 = new Page(createLongSequenceBlock(1, POSITIONS_PER_PAGE)); Page page3 = new Page(createLongSequenceBlock(2, POSITIONS_PER_PAGE)); List expected = readLongs(Stream.of(page1, page2, page3), 0); - processPages(pagePartitioner, partitioningMode, page1, page2, page3); + processPages(multiPartitionPartitioner, partitioningMode, page1, page2, page3); List partitioned = readLongs(outputBuffer.getEnqueuedDeserialized(), 0); assertThat(partitioned).containsExactlyInAnyOrderElementsOf(expected); // order is different due to 2 partitions joined + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + OperatorContext operatorContext = operatorContext(); + singlePartitionPartitioner.partitionPage(page1, operatorContext); + singlePartitionPartitioner.partitionPage(page2, operatorContext); + singlePartitionPartitioner.partitionPage(page3, operatorContext); + singlePartitionPartitioner.close(); + List singlePartitionResult = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionResult).containsExactlyElementsOf(expected); } @Test @@ -223,15 +253,25 @@ public void testOutputForSimplePageWithReplication() private void testOutputForSimplePageWithReplication(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).replicate().build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).replicate().build(); Page page = new Page(createLongsBlock(0L, 1L, 2L, 3L, null)); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0).containsExactly(0L, 2L, null); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 0); assertThat(partition1).containsExactly(0L, 1L, 3L); // position 0 copied to all partitions + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .replicate() + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionResult = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionResult).containsExactly(0L, 1L, 2L, 3L, null); } @Test @@ -244,15 +284,25 @@ public void testOutputForSimplePageWithNullChannel() private void testOutputForSimplePageWithNullChannel(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).withNullChannel(0).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).withNullChannel(0).build(); Page page = new Page(createLongsBlock(0L, 1L, 2L, 3L, null)); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0).containsExactlyInAnyOrder(0L, 2L, null); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 0); assertThat(partition1).containsExactlyInAnyOrder(1L, 3L, null); // null copied to all partitions + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withNullChannel(0) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionResult = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionResult).containsExactly(0L, 1L, 2L, 3L, null); } @Test @@ -265,19 +315,30 @@ public void testOutputForSimplePageWithPartitionConstant() private void testOutputForSimplePageWithPartitionConstant(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT) + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) .withPartitionConstants(ImmutableList.of(Optional.of(new NullableValue(BIGINT, 1L)))) .withPartitionChannels(-1) .build(); Page page = new Page(createLongsBlock(0L, 1L, 2L, 3L, null)); List allValues = readLongs(Stream.of(page), 0); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0).isEmpty(); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 0); assertThat(partition1).containsExactlyElementsOf(allValues); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionConstants(ImmutableList.of(Optional.of(new NullableValue(BIGINT, 1L)))) + .withPartitionChannels(-1) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionResult = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionResult).containsExactlyElementsOf(allValues); } @Test @@ -290,19 +351,31 @@ public void testOutputForSimplePageWithPartitionConstantAndHashBlock() private void testOutputForSimplePageWithPartitionConstantAndHashBlock(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT) + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) .withPartitionConstants(ImmutableList.of(Optional.empty(), Optional.of(new NullableValue(BIGINT, 1L)))) .withPartitionChannels(0, -1) // use first block and constant block at index 1 as input to partitionFunction .withHashChannels(0, 1) // use both channels to calculate partition (a+b) mod 2 .build(); Page page = new Page(createLongsBlock(0L, 1L, 2L, 3L)); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0).containsExactly(1L, 3L); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 0); assertThat(partition1).containsExactly(0L, 2L); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionConstants(ImmutableList.of(Optional.empty(), Optional.of(new NullableValue(BIGINT, 1L)))) + .withPartitionChannels(0, -1) // use first block and constant block at index 1 as input to partitionFunction + .withHashChannels(0, 1) // use both channels to calculate partition (a+b) mod 2 + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionChannelZero = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionChannelZero).containsExactly(0L, 1L, 2L, 3L); } @Test @@ -315,16 +388,27 @@ public void testPartitionPositionsWithRleNotNull() private void testPartitionPositionsWithRleNotNull(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT).build(); Page page = new Page(createRepeatedValuesBlock(0, POSITIONS_PER_PAGE), createLongSequenceBlock(0, POSITIONS_PER_PAGE)); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 1); assertThat(partition0).containsExactlyElementsOf(readLongs(Stream.of(page), 1)); List partition0HashBlock = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0HashBlock).containsOnly(0L).hasSize(POSITIONS_PER_PAGE); assertThat(outputBuffer.getEnqueuedDeserialized(1)).isEmpty(); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionChannelZero = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionChannelZero).containsExactlyElementsOf(readLongs(Stream.of(page), 0)); + List singlePartitionChannelOne = readLongs(outputBuffer.getEnqueuedDeserialized(0), 1); + assertThat(singlePartitionChannelOne).containsExactlyElementsOf(readLongs(Stream.of(page), 1)); } @Test @@ -337,15 +421,27 @@ public void testPartitionPositionsWithRleNotNullWithReplication() private void testPartitionPositionsWithRleNotNullWithReplication(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT).replicate().build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT).replicate().build(); Page page = new Page(createRepeatedValuesBlock(0, POSITIONS_PER_PAGE), createLongSequenceBlock(0, POSITIONS_PER_PAGE)); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 1); assertThat(partition0).containsExactlyElementsOf(readLongs(Stream.of(page), 1)); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 1); assertThat(partition1).containsExactly(0L); // position 0 copied to all partitions + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT) + .replicate() + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singePartitionChannelZero = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singePartitionChannelZero).containsExactlyElementsOf(readLongs(Stream.of(page), 0)); + List singlePartitionChannelOne = readLongs(outputBuffer.getEnqueuedDeserialized(0), 1); + assertThat(singlePartitionChannelOne).containsExactlyElementsOf(readLongs(Stream.of(page), 1)); } @Test @@ -358,15 +454,27 @@ public void testPartitionPositionsWithRleNullWithNullChannel() private void testPartitionPositionsWithRleNullWithNullChannel(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT).withNullChannel(0).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT).withNullChannel(0).build(); Page page = new Page(RunLengthEncodedBlock.create(createLongsBlock((Long) null), POSITIONS_PER_PAGE), createLongSequenceBlock(0, POSITIONS_PER_PAGE)); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 1); assertThat(partition0).containsExactlyElementsOf(readLongs(Stream.of(page), 1)); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 1); assertThat(partition1).containsExactlyElementsOf(readLongs(Stream.of(page), 1)); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT, BIGINT) + .withNullChannel(0) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionChannelZero = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); + assertThat(singlePartitionChannelZero).containsExactlyElementsOf(readLongs(Stream.of(page), 0)); + List singlePartitionChannelOne = readLongs(outputBuffer.getEnqueuedDeserialized(0), 1); + assertThat(singlePartitionChannelOne).containsExactlyElementsOf(readLongs(Stream.of(page), 1)); } @Test @@ -379,15 +487,24 @@ public void testOutputForDictionaryBlock() private void testOutputForDictionaryBlock(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).build(); Page page = new Page(createLongDictionaryBlock(0, 10)); // must have at least 10 position to have non-trivial dict - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0).containsExactlyElementsOf(nCopies(5, 0L)); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 0); assertThat(partition1).containsExactlyElementsOf(nCopies(5, 1L)); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionedOutput = readChannel(outputBuffer.getEnqueuedDeserialized(0), 0, BIGINT); + assertThat(singlePartitionedOutput).containsExactlyElementsOf(readChannel(Stream.of(page), 0, BIGINT)); } @Test @@ -400,15 +517,24 @@ public void testOutputForOneValueDictionaryBlock() private void testOutputForOneValueDictionaryBlock(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).build(); Page page = new Page(DictionaryBlock.create(4, createLongsBlock(0), new int[] {0, 0, 0, 0})); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0).containsExactlyElementsOf(nCopies(4, 0L)); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 0); assertThat(partition1).isEmpty(); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionOutput = readChannel(outputBuffer.getEnqueuedDeserialized(0), 0, BIGINT); + assertThat(singlePartitionOutput).containsExactlyElementsOf(readChannel(Stream.of(page), 0, BIGINT)); } @Test @@ -421,15 +547,24 @@ public void testOutputForViewDictionaryBlock() private void testOutputForViewDictionaryBlock(PartitioningMode partitioningMode) { TestOutputBuffer outputBuffer = new TestOutputBuffer(); - PagePartitioner pagePartitioner = pagePartitioner(outputBuffer, BIGINT).build(); + PagePartitioner multiPartitionPartitioner = pagePartitioner(outputBuffer, BIGINT).build(); Page page = new Page(DictionaryBlock.create(4, createLongSequenceBlock(4, 8), new int[] {1, 0, 3, 2})); - processPages(pagePartitioner, partitioningMode, page); + processPages(multiPartitionPartitioner, partitioningMode, page); List partition0 = readLongs(outputBuffer.getEnqueuedDeserialized(0), 0); assertThat(partition0).containsExactlyInAnyOrder(4L, 6L); List partition1 = readLongs(outputBuffer.getEnqueuedDeserialized(1), 0); assertThat(partition1).containsExactlyInAnyOrder(5L, 7L); + outputBuffer.clear(); + + PagePartitioner singlePartitionPartitioner = pagePartitioner(outputBuffer, BIGINT) + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + singlePartitionPartitioner.partitionPage(page, operatorContext()); + singlePartitionPartitioner.close(); + List singlePartitionOutput = readChannel(outputBuffer.getEnqueuedDeserialized(0), 0, BIGINT); + assertThat(singlePartitionOutput).containsExactlyElementsOf(readChannel(Stream.of(page), 0, BIGINT)); } @Test @@ -595,7 +730,7 @@ private void testOutputEqualsInput(Type type, PartitioningMode mode1, Partitioni { TestOutputBuffer outputBuffer = new TestOutputBuffer(); PagePartitionerBuilder pagePartitionerBuilder = pagePartitioner(outputBuffer, BIGINT, type, type); - PagePartitioner pagePartitioner = pagePartitionerBuilder.build(); + PagePartitioner multiPartitionPartitioner = pagePartitionerBuilder.build(); Page input = new Page( createLongSequenceBlock(0, POSITIONS_PER_PAGE), // partition block createBlockForType(type, POSITIONS_PER_PAGE), @@ -603,14 +738,25 @@ private void testOutputEqualsInput(Type type, PartitioningMode mode1, Partitioni List expected = readChannel(Stream.of(input, input), 1, type); - mode1.partitionPage(pagePartitioner, input); - mode2.partitionPage(pagePartitioner, input); + mode1.partitionPage(multiPartitionPartitioner, input); + mode2.partitionPage(multiPartitionPartitioner, input); - pagePartitioner.close(); + multiPartitionPartitioner.close(); - List partitioned = readChannel(outputBuffer.getEnqueuedDeserialized(), 1, type); - assertThat(partitioned).containsExactlyInAnyOrderElementsOf(expected); // output of the PagePartitioner can be reordered + List multiPartitionedOutput = readChannel(outputBuffer.getEnqueuedDeserialized(), 1, type); + assertThat(multiPartitionedOutput).containsExactlyInAnyOrderElementsOf(expected); // output of the PagePartitioner can be reordered outputBuffer.clear(); + + // Test single partition output matches input + PagePartitioner singlePartitionPartitioner = pagePartitionerBuilder + .withPartitionFunction(new SinglePartitionFailIfCalled()) + .build(); + OperatorContext operatorContext = operatorContext(); + singlePartitionPartitioner.partitionPage(input, operatorContext); + singlePartitionPartitioner.partitionPage(input, operatorContext); + singlePartitionPartitioner.close(); + List singlePartitionedOutput = readChannel(outputBuffer.getEnqueuedDeserialized(), 1, type); + assertThat(singlePartitionedOutput).isEqualTo(expected); } private static Block createBlockForType(Type type, int positionsPerPage) @@ -973,4 +1119,20 @@ public int getPartition(Page page, int position) return toIntExact(Math.abs(value) % partitionCount); } } + + private static final class SinglePartitionFailIfCalled + implements PartitionFunction + { + @Override + public int partitionCount() + { + return 1; + } + + @Override + public int getPartition(Page page, int position) + { + throw new UnsupportedOperationException("getPartition should not be called on single partitioned outputs"); + } + } } diff --git a/core/trino-main/src/test/java/io/trino/operator/output/TestPositionsAppender.java b/core/trino-main/src/test/java/io/trino/operator/output/TestPositionsAppender.java index 453a51a3a1dc..a0218fc0e6ff 100644 --- a/core/trino-main/src/test/java/io/trino/operator/output/TestPositionsAppender.java +++ b/core/trino-main/src/test/java/io/trino/operator/output/TestPositionsAppender.java @@ -43,6 +43,7 @@ import it.unimi.dsi.fastutil.ints.IntArrayList; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -498,6 +499,7 @@ private static void testAppend(TestType type, List inputs) { testAppendBatch(type, inputs); testAppendSingle(type, inputs); + testAppendRange(type, inputs); } private static void testAppendBatch(TestType type, List inputs) @@ -509,6 +511,19 @@ private static void testAppendBatch(TestType type, List inputs) assertBuildResult(type, inputs, positionsAppender, initialRetainedSize); } + private static void testAppendRange(TestType type, List inputs) + { + UnnestingPositionsAppender positionsAppender = POSITIONS_APPENDER_FACTORY.create(type.getType(), 10, DEFAULT_MAX_PAGE_SIZE_IN_BYTES); + long initialRetainedSize = positionsAppender.getRetainedSizeInBytes(); + + inputs.forEach(input -> { + for (PositionRange range : computeRanges(input.positions())) { + positionsAppender.appendRange(input.block(), range.offset(), range.length()); + } + }); + assertBuildResult(type, inputs, positionsAppender, initialRetainedSize); + } + private static void assertBuildResult(TestType type, List inputs, UnnestingPositionsAppender positionsAppender, long initialRetainedSize) { long sizeInBytes = positionsAppender.getSizeInBytes(); @@ -555,8 +570,9 @@ private static Block buildBlock(Type type, List inputs, BlockBuilderS { BlockBuilder blockBuilder = type.createBlockBuilder(blockBuilderStatus, 10); for (BlockView input : inputs) { + ValueBlock valueBlock = input.block().getUnderlyingValueBlock(); for (int position : input.positions()) { - type.appendTo(input.block(), position, blockBuilder); + blockBuilder.append(valueBlock, input.block().getUnderlyingValuePosition(position)); } } return blockBuilder.build(); @@ -614,6 +630,47 @@ private record BlockView(Block block, IntArrayList positions) } } + private record PositionRange(int offset, int length) + { + private PositionRange + { + checkArgument(offset >= 0, "offset must be >= 0, found: %s", offset); + checkArgument(length > 0, "length must be positive, found: %s", length); + } + } + + private static List computeRanges(IntArrayList positions) + { + List ranges = new ArrayList<>(); + int start = 0; + while (start < positions.size()) { + int position = positions.getInt(start); + int length = 1; + while (start + length < positions.size() && position + length == positions.getInt(start + length)) { + length++; + } + ranges.add(new PositionRange(position, length)); + start += length; + } + return List.copyOf(ranges); + } + + @Test + public void testComputeRanges() + { + assertThat(computeRanges(allPositions(10))).isEqualTo(List.of(new PositionRange(0, 10))); + assertThat(computeRanges(IntArrayList.of(0, 2, 4, 6))) + .isEqualTo(List.of( + new PositionRange(0, 1), + new PositionRange(2, 1), + new PositionRange(4, 1), + new PositionRange(6, 1))); + assertThat(computeRanges(IntArrayList.of(1, 2, 4, 5))) + .isEqualTo(List.of( + new PositionRange(1, 2), + new PositionRange(4, 2))); + } + private static Function adaptation() { return TestPositionsAppender::adapt; diff --git a/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayDistinct.java b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayDistinct.java index 5cde29b69004..7064d9b8c6f7 100644 --- a/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayDistinct.java +++ b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayDistinct.java @@ -23,6 +23,7 @@ import io.trino.spi.block.ArrayBlockBuilder; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.connector.SourcePage; import io.trino.spi.function.ScalarFunction; import io.trino.spi.function.SqlType; @@ -177,11 +178,12 @@ public static Block oldArrayDistinct(@SqlType("array(varchar)") Block array) return array; } - BlockSet set = new BlockSet(VARCHAR, DISTINCT_FROM_OPERATOR, HASH_CODE_OPERATOR, array.getPositionCount()); + BlockSet set = new BlockSet(DISTINCT_FROM_OPERATOR, HASH_CODE_OPERATOR, array.getPositionCount()); BlockBuilder distinctElementBlockBuilder = VARCHAR.createBlockBuilder(null, array.getPositionCount()); + ValueBlock valueBlock = array.getUnderlyingValueBlock(); for (int i = 0; i < array.getPositionCount(); i++) { if (set.add(array, i)) { - VARCHAR.appendTo(array, i, distinctElementBlockBuilder); + distinctElementBlockBuilder.append(valueBlock, array.getUnderlyingValuePosition(i)); } } diff --git a/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayFilter.java b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayFilter.java index 69cf79286730..fac92b4ea9df 100644 --- a/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayFilter.java +++ b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArrayFilter.java @@ -25,6 +25,7 @@ import io.trino.spi.block.ArrayBlockBuilder; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.connector.SourcePage; import io.trino.spi.function.BoundSignature; import io.trino.spi.function.FunctionMetadata; @@ -307,6 +308,7 @@ protected SpecializedSqlScalarFunction specialize(BoundSignature boundSignature) public static Block filter(Type type, Block block, MethodHandle function) { int positionCount = block.getPositionCount(); + ValueBlock valueBlock = block.getUnderlyingValueBlock(); BlockBuilder resultBuilder = type.createBlockBuilder(null, positionCount); for (int position = 0; position < positionCount; position++) { Long input = (Long) readNativeValue(type, block, position); @@ -319,7 +321,7 @@ public static Block filter(Type type, Block block, MethodHandle function) throw new RuntimeException(t); } if (TRUE.equals(keep)) { - type.appendTo(block, position, resultBuilder); + resultBuilder.append(valueBlock, block.getUnderlyingValuePosition(position)); } } return resultBuilder.build(); @@ -362,6 +364,7 @@ public static Block filterObject(Type type, Block block, MethodHandle function) { int positionCount = block.getPositionCount(); BlockBuilder resultBuilder = type.createBlockBuilder(null, positionCount); + ValueBlock valueBlock = block.getUnderlyingValueBlock(); for (int position = 0; position < positionCount; position++) { Object input = type.getObject(block, position); Boolean keep; @@ -373,7 +376,7 @@ public static Block filterObject(Type type, Block block, MethodHandle function) throw new RuntimeException(t); } if (TRUE.equals(keep)) { - type.appendTo(block, position, resultBuilder); + resultBuilder.append(valueBlock, block.getUnderlyingValuePosition(position)); } } return resultBuilder.build(); diff --git a/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArraySort.java b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArraySort.java index e52108dfdcce..6afaa846e059 100644 --- a/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArraySort.java +++ b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkArraySort.java @@ -24,6 +24,7 @@ import io.trino.spi.block.ArrayBlockBuilder; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; +import io.trino.spi.block.ValueBlock; import io.trino.spi.connector.SourcePage; import io.trino.spi.function.ScalarFunction; import io.trino.spi.function.SqlType; @@ -182,9 +183,9 @@ public static Block oldArraySort(@SqlType("array(varchar)") Block block) }); BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, block.getPositionCount()); - + ValueBlock valueBlock = block.getUnderlyingValueBlock(); for (int position : positions) { - VARCHAR.appendTo(block, position, blockBuilder); + blockBuilder.append(valueBlock, block.getUnderlyingValuePosition(position)); } return blockBuilder.build(); diff --git a/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkHistogram.java b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkHistogram.java new file mode 100644 index 000000000000..e90b0eacb1d7 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/operator/scalar/BenchmarkHistogram.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.trino.operator.scalar; + +import io.airlift.stats.TDigest; +import io.trino.jmh.Benchmarks; +import io.trino.plugin.base.metrics.DistributionSnapshot; +import io.trino.plugin.base.metrics.TDigestHistogram; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.security.SecureRandom; +import java.util.concurrent.TimeUnit; + +@SuppressWarnings("MethodMayBeStatic") +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(1) +@Warmup(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@BenchmarkMode(Mode.AverageTime) +public class BenchmarkHistogram +{ + private static final int HISTOGRAM_VALUES_SETS = 100; + + @Benchmark + @OperationsPerInvocation(HISTOGRAM_VALUES_SETS) + public void benchmarkTDigestSnapshot(BenchmarkData data, Blackhole bh) + { + for (TDigest digest : data.tDigests()) { + bh.consume(DistributionSnapshot.fromDistribution(new TDigestHistogram(digest))); + } + } + + @SuppressWarnings("FieldMayBeFinal") + @State(Scope.Thread) + public static class BenchmarkData + { + @Param({"10", "100", "1000"}) + private int valuesPerHistogram = 100; + + TDigest[] tDigests; + + @Setup + public void setup() + { + SecureRandom random = new SecureRandom(); + tDigests = new TDigest[HISTOGRAM_VALUES_SETS]; + for (int i = 0; i < HISTOGRAM_VALUES_SETS; i++) { + TDigest digest = new TDigest(); + for (int j = 0; j < valuesPerHistogram; j++) { + long value = random.nextLong(10_000_000, 10_000_000_000L); + digest.add(value); + } + tDigests[i] = digest; + } + } + + public TDigest[] tDigests() + { + return tDigests; + } + } + + public static void main(String[] args) + throws Exception + { + // assure the benchmarks are valid before running + BenchmarkData data = new BenchmarkData(); + data.setup(); + Blackhole bh = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous."); + new BenchmarkHistogram().benchmarkTDigestSnapshot(data, bh); + + Benchmarks.benchmark(BenchmarkHistogram.class).run(); + } +} diff --git a/core/trino-main/src/test/java/io/trino/operator/scalar/TestBlockSet.java b/core/trino-main/src/test/java/io/trino/operator/scalar/TestBlockSet.java index faf8cdd7eb2f..dd398941dc9f 100644 --- a/core/trino-main/src/test/java/io/trino/operator/scalar/TestBlockSet.java +++ b/core/trino-main/src/test/java/io/trino/operator/scalar/TestBlockSet.java @@ -53,7 +53,7 @@ public void testConstructor() .hasMessage("maximumSize must not be negative"); } - assertThatThrownBy(() -> new BlockSet(null, null, null, 1)) + assertThatThrownBy(() -> new BlockSet(null, null, 1)) .isInstanceOfAny(NullPointerException.class, IllegalArgumentException.class); } @@ -264,7 +264,6 @@ public void testMemoryExceeded() private static BlockSet createBlockSet(Type type, int expectedSize) { return new BlockSet( - type, BLOCK_TYPE_OPERATORS.getIdenticalOperator(type), BLOCK_TYPE_OPERATORS.getHashCodeOperator(type), expectedSize); diff --git a/core/trino-main/src/test/java/io/trino/operator/spiller/BenchmarkBinaryFileSpiller.java b/core/trino-main/src/test/java/io/trino/operator/spiller/BenchmarkBinaryFileSpiller.java index 466203fc1ee4..aa4482456149 100644 --- a/core/trino-main/src/test/java/io/trino/operator/spiller/BenchmarkBinaryFileSpiller.java +++ b/core/trino-main/src/test/java/io/trino/operator/spiller/BenchmarkBinaryFileSpiller.java @@ -117,6 +117,7 @@ public void setup() BLOCK_ENCODING_SERDE, spillerStats, ImmutableList.of(SPILL_PATH), + 1, 1.0, compressionCodec, encryptionEnabled); diff --git a/core/trino-main/src/test/java/io/trino/operator/unnest/TestingUnnesterUtil.java b/core/trino-main/src/test/java/io/trino/operator/unnest/TestingUnnesterUtil.java index d3cd41bb77d0..578933c4e7e1 100644 --- a/core/trino-main/src/test/java/io/trino/operator/unnest/TestingUnnesterUtil.java +++ b/core/trino-main/src/test/java/io/trino/operator/unnest/TestingUnnesterUtil.java @@ -182,7 +182,7 @@ static Page mergePages(List types, List pages) blockBuilder.appendNull(); } else { - types.get(i).appendTo(block, position, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); } } } @@ -199,7 +199,7 @@ private static Block buildExpectedReplicatedBlock(Block block, Type type, int[] for (int i = 0; i < positionCount; i++) { int cardinality = maxCardinalities[i]; for (int j = 0; j < cardinality; j++) { - type.appendTo(block, i, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(i)); } } return blockBuilder.build(); @@ -216,7 +216,7 @@ private static Block buildExpectedUnnestedArrayBlock(Block block, Type type, int for (int i = 0; i < positionCount; i++) { int cardinality = columnarArray.getLength(i); for (int j = 0; j < cardinality; j++) { - type.appendTo(elementBlock, elementBlockPosition++, blockBuilder); + blockBuilder.append(elementBlock.getUnderlyingValueBlock(), elementBlock.getUnderlyingValuePosition(elementBlockPosition++)); } int maxCardinality = maxCardinalities[i]; @@ -241,8 +241,8 @@ private static Block[] buildExpectedUnnestedMapBlocks(Block block, Type keyType, for (int i = 0; i < positionCount; i++) { int cardinality = columnarMap.getEntryCount(i); for (int j = 0; j < cardinality; j++) { - keyType.appendTo(keyBlock, blockPosition, keyBlockBuilder); - valueType.appendTo(valuesBlock, blockPosition, valueBlockBuilder); + keyBlockBuilder.append(keyBlock.getUnderlyingValueBlock(), keyBlock.getUnderlyingValuePosition(blockPosition)); + valueBlockBuilder.append(valuesBlock.getUnderlyingValueBlock(), valuesBlock.getUnderlyingValuePosition(blockPosition)); blockPosition++; } @@ -274,7 +274,8 @@ private static Block[] buildExpectedUnnestedArrayOfRowBlock(Block block, List assertThat(nodeStateManager.getServerState()).isEqualTo(DRAINED)); + await().atMost(5, SECONDS).untilAsserted(() -> assertThat(nodeStateManager.getServerState()).isEqualTo(DRAINED)); } @Test @@ -105,8 +105,8 @@ void testTransitionToShuttingDown() nodeStateManager.transitionState(NodeState.SHUTTING_DOWN); assertThat(nodeStateManager.getServerState()).isEqualTo(NodeState.SHUTTING_DOWN); - // here wait for at least 2 grace periods, and add some slack to reduce test flakyness - await().atMost(4 * GRACE_PERIOD_MILLIS + 100, MILLISECONDS).until(() -> shutdownAction.isShuttingDown()); + // here wait for at least 4 grace periods, and add some slack to reduce test flakiness + await().atMost(4 * GRACE_PERIOD_MILLIS + 1000, MILLISECONDS).until(() -> shutdownAction.isShuttingDown()); } @Test @@ -117,7 +117,7 @@ void testCannotReactivateShuttingDown() nodeStateManager.transitionState(NodeState.SHUTTING_DOWN); assertThat(nodeStateManager.getServerState()).isEqualTo(NodeState.SHUTTING_DOWN); - // here wait for at least 2 grace periods, and add some slack to reduce test flakyness + // here wait for at least 4 grace periods, and add some slack to reduce test flakiness await().atMost(4 * GRACE_PERIOD_MILLIS, MILLISECONDS).until(() -> shutdownAction.isShuttingDown()); assertThatThrownBy(() -> nodeStateManager.transitionState(ACTIVE)) @@ -137,16 +137,15 @@ void testImmediateTransitionToShuttingDownWhenDrained() ticker.increment(1, SECONDS); executor.run(); // 2 gracePeriods or more - await().atMost(2 * GRACE_PERIOD_MILLIS + 100, SECONDS) + await().atMost(2 * GRACE_PERIOD_MILLIS + 100, MILLISECONDS) .untilAsserted(() -> assertThat(nodeStateManager.getServerState()).isEqualTo(DRAINED)); // now test the shutdown from the DRAINED state nodeStateManager.transitionState(NodeState.SHUTTING_DOWN); assertThat(nodeStateManager.getServerState()).isEqualTo(NodeState.SHUTTING_DOWN); - // here only wait for minimal amount of time, as shutdown should be immediate await().pollInterval(1, MILLISECONDS) - .atMost(100, MILLISECONDS).until(() -> shutdownAction.isShuttingDown()); + .atMost(1, SECONDS).until(() -> shutdownAction.isShuttingDown()); } @Test @@ -171,7 +170,7 @@ void testWaitActiveTasksToFinishDuringShutdown() // make sure that nodeStateManager registered a listener for tasks to finish ticker.increment(1, SECONDS); executor.run(); - await().atMost(1, SECONDS).until(() -> sqlTasksObservable.getTasks().size() == 1); + await().atMost(5, SECONDS).until(() -> sqlTasksObservable.getTasks().size() == 1); // simulate task completion after some time tasks.set(Collections.emptyList()); @@ -213,7 +212,7 @@ void testWaitActiveTasksToFinishDuringDraining() .stateChanged(TaskState.FINISHED); // when NodeStateManager sees task finished - it will drain after another drain period - await().atMost(1, SECONDS) + await().atMost(5, SECONDS) .untilAsserted(() -> assertThat(nodeStateManager.getServerState()).isEqualTo(DRAINED)); } diff --git a/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java b/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java index 4e7c5bbfe0b1..061293dd9ee9 100644 --- a/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java +++ b/core/trino-main/src/test/java/io/trino/server/TestQueryStateInfo.java @@ -179,6 +179,7 @@ private QueryInfo createQueryInfo(String queryId, QueryState state, String query DataSize.valueOf("41GB"), ImmutableList.of(), DynamicFiltersStats.EMPTY, + ImmutableMap.of(), ImmutableList.of(), ImmutableList.of()), Optional.empty(), diff --git a/core/trino-main/src/test/java/io/trino/server/TestServerPluginsProviderConfig.java b/core/trino-main/src/test/java/io/trino/server/TestServerPluginsProviderConfig.java index c88a404ac1c1..ebb26c96505c 100644 --- a/core/trino-main/src/test/java/io/trino/server/TestServerPluginsProviderConfig.java +++ b/core/trino-main/src/test/java/io/trino/server/TestServerPluginsProviderConfig.java @@ -15,8 +15,10 @@ import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; -import java.io.File; +import java.nio.file.Path; +import java.util.List; import java.util.Map; import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; @@ -29,16 +31,27 @@ public class TestServerPluginsProviderConfig public void testDefaults() { assertRecordedDefaults(recordDefaults(ServerPluginsProviderConfig.class) - .setInstalledPluginsDir(new File("plugin"))); + .setInstalledPluginsDirs(List.of(Path.of("plugin")))); } @Test - public void testExplicitPropertyMappings() + public void testExplicitPropertyMappings(@TempDir Path tempDir) { - Map properties = ImmutableMap.of("plugin.dir", "plugins-dir"); + Map properties = ImmutableMap.of("plugin.dir", tempDir.toString()); ServerPluginsProviderConfig expected = new ServerPluginsProviderConfig() - .setInstalledPluginsDir(new File("plugins-dir")); + .setInstalledPluginsDirs(List.of(tempDir)); + + assertFullMapping(properties, expected); + } + + @Test + public void testExplicitPropertyMappingMultiDir(@TempDir Path tempDir1, @TempDir Path tempDir2) + { + Map properties = ImmutableMap.of("plugin.dir", tempDir1.toString() + "," + tempDir2.toString()); + + ServerPluginsProviderConfig expected = new ServerPluginsProviderConfig() + .setInstalledPluginsDirs(List.of(tempDir1, tempDir2)); assertFullMapping(properties, expected); } diff --git a/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java b/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java index 323b37223097..9d46eca7ddfd 100644 --- a/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java +++ b/core/trino-main/src/test/java/io/trino/server/remotetask/TestHttpRemoteTask.java @@ -448,7 +448,7 @@ public void testOutboundDynamicFilters() // make sure initial dynamic filter is collected CompletableFuture future = dynamicFilter.isBlocked(); dynamicFilterService.addTaskDynamicFilters( - new TaskId(new StageId(queryId.getId(), 1), 1, 0), + new TaskId(new StageId(queryId.id(), 1), 1, 0), ImmutableMap.of(filterId1, Domain.singleValue(BIGINT, 1L))); future.get(); assertThat(dynamicFilter.getCurrentPredicate()).isEqualTo(TupleDomain.withColumnDomains(ImmutableMap.of( @@ -474,7 +474,7 @@ public void testOutboundDynamicFilters() future = dynamicFilter.isBlocked(); dynamicFilterService.addTaskDynamicFilters( - new TaskId(new StageId(queryId.getId(), 1), 1, 0), + new TaskId(new StageId(queryId.id(), 1), 1, 0), ImmutableMap.of(filterId2, Domain.singleValue(BIGINT, 2L))); future.get(); assertThat(dynamicFilter.getCurrentPredicate()).isEqualTo(TupleDomain.withColumnDomains(ImmutableMap.of( diff --git a/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java b/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java index c16fad53035b..c3c2cd01d9e9 100644 --- a/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java +++ b/core/trino-main/src/test/java/io/trino/server/security/oauth2/TestingHydraIdentityProvider.java @@ -48,10 +48,11 @@ import org.testcontainers.containers.FixedHostPortGenericContainer; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.containers.wait.strategy.WaitAllStrategy; +import org.testcontainers.postgresql.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; import java.io.IOException; @@ -71,13 +72,15 @@ public class TestingHydraIdentityProvider implements AutoCloseable { + private static final DockerImageName POSTGRES_IMAGE = DockerImageName.parse(PostgreSQLContainer.IMAGE) + .withTag(PostgreSQLContainer.DEFAULT_TAG); private static final String HYDRA_IMAGE = "oryd/hydra:v1.11.10"; private static final String ISSUER = "https://localhost:4444/"; private static final String DSN = "postgres://hydra:mysecretpassword@database:5432/hydra?sslmode=disable"; private final Network network = Network.newNetwork(); - private final PostgreSQLContainer databaseContainer = new PostgreSQLContainer<>() + private final PostgreSQLContainer databaseContainer = new PostgreSQLContainer(POSTGRES_IMAGE) .withNetwork(network) .withNetworkAliases("database") .withUsername("hydra") diff --git a/core/trino-main/src/test/java/io/trino/spiller/TestBinaryFileSpiller.java b/core/trino-main/src/test/java/io/trino/spiller/TestBinaryFileSpiller.java index 7eaf1e2ddc0e..842d8413c626 100644 --- a/core/trino-main/src/test/java/io/trino/spiller/TestBinaryFileSpiller.java +++ b/core/trino-main/src/test/java/io/trino/spiller/TestBinaryFileSpiller.java @@ -17,6 +17,7 @@ import io.airlift.slice.Slices; import io.trino.FeaturesConfig; import io.trino.RowPagesBuilder; +import io.trino.execution.TaskManagerConfig; import io.trino.execution.buffer.PageSerializer; import io.trino.execution.buffer.PagesSerdeFactory; import io.trino.memory.context.AggregatedMemoryContext; @@ -82,7 +83,7 @@ public void setUp() featuresConfig.setSpillMaxUsedSpaceThreshold(1.0); NodeSpillConfig nodeSpillConfig = new NodeSpillConfig(); BlockEncodingSerde blockEncodingSerde = new TestingBlockEncodingSerde(); - singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(blockEncodingSerde, spillerStats, featuresConfig, nodeSpillConfig); + singleStreamSpillerFactory = new FileSingleStreamSpillerFactory(blockEncodingSerde, spillerStats, featuresConfig, nodeSpillConfig, new TaskManagerConfig()); factory = new GenericSpillerFactory(singleStreamSpillerFactory); PagesSerdeFactory pagesSerdeFactory = createSpillingPagesSerdeFactory(blockEncodingSerde, nodeSpillConfig.getSpillCompressionCodec()); serializer = pagesSerdeFactory.createSerializer(Optional.empty()); @@ -162,7 +163,7 @@ private void testSpiller(List types, Spiller spiller, List... spills assertThat(spillerStats.getTotalSpilledBytes() - spilledBytesBefore).isEqualTo(spilledBytes); // At this point, the buffers should still be accounted for in the memory context, because // the spiller (FileSingleStreamSpiller) doesn't release its memory reservation until it's closed. - assertThat(memoryContext.getBytes()).isEqualTo((long) spills.length * FileSingleStreamSpiller.BUFFER_SIZE); + assertThat(memoryContext.getBytes()).isEqualTo((long) spills.length * SpillFile.BUFFER_SIZE); List> actualSpills = spiller.getSpills(); assertThat(actualSpills).hasSize(spills.length); diff --git a/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpiller.java b/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpiller.java index 2b8f829ca99a..13a8cac50f0f 100644 --- a/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpiller.java +++ b/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpiller.java @@ -18,7 +18,11 @@ import com.google.common.util.concurrent.ListeningExecutorService; import io.airlift.slice.Slice; import io.airlift.slice.Slices; +import io.airlift.units.DataSize; import io.trino.execution.buffer.CompressionCodec; +import io.trino.execution.buffer.PageDeserializer; +import io.trino.execution.buffer.PageSerializer; +import io.trino.execution.buffer.PagesSerdeFactory; import io.trino.execution.buffer.PagesSerdeUtil; import io.trino.memory.context.LocalMemoryContext; import io.trino.operator.PageAssertions; @@ -34,17 +38,22 @@ import java.io.File; import java.io.InputStream; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import java.util.Optional; +import java.util.function.Function; import static com.google.common.io.MoreFiles.deleteRecursively; import static com.google.common.io.MoreFiles.listFiles; import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE; import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; +import static io.airlift.concurrent.MoreFutures.getFutureValue; import static io.trino.execution.buffer.CompressionCodec.LZ4; import static io.trino.execution.buffer.CompressionCodec.NONE; import static io.trino.execution.buffer.PagesSerdeUtil.isSerializedPageCompressed; import static io.trino.execution.buffer.PagesSerdeUtil.isSerializedPageEncrypted; +import static io.trino.execution.buffer.PagesSerdes.createSpillingPagesSerdeFactory; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.spi.type.DoubleType.DOUBLE; @@ -99,6 +108,41 @@ public void testSpillEncryptionWithCompression() assertSpill(LZ4, true); } + @Test + public void testMultiFileSpill() + throws Exception + { + // Writes four pages in a single call and verifies spilled bytes, non-empty spill files, + // and round-robin ordering when reading the pages back. + assertMultiFileSpill(spiller -> ImmutableList.copyOf(spiller.getSpilledPages()), false); + } + + @Test + public void testMultiFileSpillMultipleCalls() + throws Exception + { + // Performs two spill() calls to ensure rotation resumes on the expected file. + assertMultiFileSpill(spiller -> ImmutableList.copyOf(spiller.getSpilledPages()), true); + } + + @Test + public void testGetAllSpilledPagesMultiFile() + throws Exception + { + // Retrieves all spilled pages asynchronously from multiple files and verifies the combined + // result preserves the round-robin write order. + assertMultiFileSpill(spiller -> getFutureValue(spiller.getAllSpilledPages()), false); + } + + @Test + public void testGetAllSpilledPagesMultiFileMultipleCalls() + throws Exception + { + // Calls getAllSpilledPages() after spilling in multiple batches to confirm partially filled + // files are preserved and the merged ordering remains correct. + assertMultiFileSpill(spiller -> getFutureValue(spiller.getAllSpilledPages()), true); + } + private void assertSpill(CompressionCodec compressionCodec, boolean encryption) throws Exception { @@ -109,6 +153,7 @@ private void assertSpill(CompressionCodec compressionCodec, boolean encryption) new TestingBlockEncodingSerde(), new SpillerStats(), ImmutableList.of(spillPath.toPath()), + 1, 1.0, compressionCodec, encryption); @@ -125,23 +170,12 @@ private void assertSpill(CompressionCodec compressionCodec, boolean encryption) spiller.spill(Iterators.forArray(page, page, page)).get(); assertThat(listFiles(spillPath.toPath())).hasSize(1); - // Assert the spill codec flags match the expected configuration - try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) { - Iterator serializedPages = PagesSerdeUtil.readSerializedPages(is); - assertThat(serializedPages.hasNext()) - .describedAs("at least one page should be successfully read back") - .isTrue(); - Slice serializedPage = serializedPages.next(); - assertThat(isSerializedPageCompressed(serializedPage)).isEqualTo(compressionCodec == LZ4); - assertThat(isSerializedPageEncrypted(serializedPage)).isEqualTo(encryption); - } - // The spillers release their memory reservations when they are closed, therefore at this point // they will have non-zero memory reservation. // assertEquals(memoryContext.getBytes(), 0); Iterator spilledPagesIterator = spiller.getSpilledPages(); - assertThat(memoryContext.getBytes()).isEqualTo(FileSingleStreamSpiller.BUFFER_SIZE); + assertThat(memoryContext.getBytes()).isEqualTo(SpillFile.BUFFER_SIZE); List spilledPages = ImmutableList.copyOf(spilledPagesIterator); // The spillers release their memory reservations when they are closed, therefore at this point // they will have non-zero memory reservation. @@ -157,6 +191,17 @@ private void assertSpill(CompressionCodec compressionCodec, boolean encryption) .isInstanceOf(IllegalStateException.class) .hasMessage("Repeated reads are disallowed to prevent potential resource leaks"); + // Assert the spill codec flags match the expected configuration + try (InputStream is = newInputStream(listFiles(spillPath.toPath()).get(0))) { + Iterator serializedPages = PagesSerdeUtil.readSerializedPages(is); + assertThat(serializedPages.hasNext()) + .describedAs("at least one page should be successfully read back") + .isTrue(); + Slice serializedPage = serializedPages.next(); + assertThat(isSerializedPageCompressed(serializedPage)).isEqualTo(compressionCodec == LZ4); + assertThat(isSerializedPageEncrypted(serializedPage)).isEqualTo(encryption); + } + spiller.close(); assertThat(listFiles(spillPath.toPath())).isEmpty(); assertThat(memoryContext.getBytes()).isEqualTo(0); @@ -166,15 +211,112 @@ private void assertSpill(CompressionCodec compressionCodec, boolean encryption) } } + private void assertMultiFileSpill(Function> readPages, boolean multipleCalls) + throws Exception + { + // Create two temporary directories to be used for spilling + File spillPath1 = Files.createTempDirectory("tmp1").toFile(); + File spillPath2 = Files.createTempDirectory("tmp2").toFile(); + try { + // Set up serializer and memory tracking objects + SpillerStats stats = new SpillerStats(); + PagesSerdeFactory serdeFactory = createSpillingPagesSerdeFactory(new TestingBlockEncodingSerde(), NONE); + LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext("test"); + PageSerializer serializer = serdeFactory.createSerializer(Optional.empty()); + PageDeserializer deserializer = serdeFactory.createDeserializer(Optional.empty()); + + // Rotate between the two files using round-robin distribution + FileSingleStreamSpiller spiller = new FileSingleStreamSpiller( + serdeFactory, + Optional.empty(), + executor, + ImmutableList.of(spillPath1.toPath(), spillPath2.toPath()), + stats, + bytes -> {}, + memoryContext, + () -> {}); + + // Build a sequence of pages with distinct numbers + Page p0 = buildPage(0); + Page p1 = buildPage(1); + Page p2 = buildPage(2); + Page p3 = buildPage(3); + + long expectedRawBytes = p0.getSizeInBytes() + p1.getSizeInBytes() + p2.getSizeInBytes() + p3.getSizeInBytes(); + long expectedSerializedBytes = + serializer.serialize(p0).length() + + serializer.serialize(p1).length() + + serializer.serialize(p2).length() + + serializer.serialize(p3).length(); + + DataSize spilled; + if (multipleCalls) { + DataSize d1 = spiller.spill(Iterators.forArray(p0)).get(); + DataSize d2 = spiller.spill(Iterators.forArray(p1, p2, p3)).get(); + spilled = DataSize.ofBytes(d1.toBytes() + d2.toBytes()); + } + else { + spilled = spiller.spill(Iterators.forArray(p0, p1, p2, p3)).get(); + } + assertThat(spilled.toBytes()).isEqualTo(expectedRawBytes); + assertThat(stats.getTotalSpilledBytes()).isEqualTo(expectedSerializedBytes); + + // Read pages using the provided function and verify order + List pages = readPages.apply(spiller); + assertThat(pages).hasSize(4); + for (int i = 0; i < pages.size(); i++) { + assertThat(BIGINT.getLong(pages.get(i).getBlock(0), 0)).isEqualTo(i); + } + + // Validate that two non-empty spill files were created + assertThat(listFiles(spillPath1.toPath()).size() + listFiles(spillPath2.toPath()).size()).isEqualTo(2); + assertThat(Files.size(listFiles(spillPath1.toPath()).get(0))).isGreaterThan(0); + assertThat(Files.size(listFiles(spillPath2.toPath()).get(0))).isGreaterThan(0); + + // Validate content distribution per spill file + try (InputStream input1 = newInputStream(listFiles(spillPath1.toPath()).get(0))) { + Iterator serializedPages1 = PagesSerdeUtil.readSerializedPages(input1); + List values = new ArrayList<>(); + while (serializedPages1.hasNext()) { + Page page = deserializer.deserialize(serializedPages1.next()); + values.add(BIGINT.getLong(page.getBlock(0), 0)); + } + assertThat(values).containsExactly(0L, 2L); + } + try (InputStream input2 = newInputStream(listFiles(spillPath2.toPath()).get(0))) { + Iterator serializedPages2 = PagesSerdeUtil.readSerializedPages(input2); + List values = new ArrayList<>(); + while (serializedPages2.hasNext()) { + Page page = deserializer.deserialize(serializedPages2.next()); + values.add(BIGINT.getLong(page.getBlock(0), 0)); + } + assertThat(values).containsExactly(1L, 3L); + } + + // Close spiller to release resources + spiller.close(); + } + finally { + // Clean up temporary directories + deleteRecursively(spillPath1.toPath(), ALLOW_INSECURE); + deleteRecursively(spillPath2.toPath(), ALLOW_INSECURE); + } + } + private Page buildPage() + { + return buildPage(42); + } + + private Page buildPage(int pageNumber) { BlockBuilder col1 = BIGINT.createFixedSizeBlockBuilder(1); BlockBuilder col2 = DOUBLE.createFixedSizeBlockBuilder(1); BlockBuilder col3 = VARBINARY.createBlockBuilder(null, 1); - BIGINT.writeLong(col1, 42); - DOUBLE.writeDouble(col2, 43.0); - VARBINARY.writeSlice(col3, Slices.allocate(16).getOutput().appendDouble(43.0).appendLong(1).slice()); + BIGINT.writeLong(col1, pageNumber); + DOUBLE.writeDouble(col2, pageNumber + 0.5); + VARBINARY.writeSlice(col3, Slices.allocate(16).getOutput().appendInt(pageNumber).appendLong(1).slice()); return new Page(col1.build(), col2.build(), col3.build()); } diff --git a/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpillerFactory.java b/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpillerFactory.java index 505e66e81486..258ec6fde546 100644 --- a/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpillerFactory.java +++ b/core/trino-main/src/test/java/io/trino/spiller/TestFileSingleStreamSpillerFactory.java @@ -279,6 +279,7 @@ private FileSingleStreamSpillerFactory spillerFactoryFactory(List paths, D blockEncodingSerde, new SpillerStats(), paths, + 1, maxUsedSpaceThreshold, NONE, false); diff --git a/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java b/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java index 09117353f94b..fb48162b71dd 100644 --- a/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java +++ b/core/trino-main/src/test/java/io/trino/spiller/TestGenericPartitioningSpiller.java @@ -19,6 +19,7 @@ import io.trino.FeaturesConfig; import io.trino.RowPagesBuilder; import io.trino.SequencePageBuilder; +import io.trino.execution.TaskManagerConfig; import io.trino.memory.context.AggregatedMemoryContext; import io.trino.operator.PartitionFunction; import io.trino.operator.SpillContext; @@ -85,7 +86,8 @@ public void setUp() new TestingBlockEncodingSerde(), new SpillerStats(), featuresConfig, - new NodeSpillConfig()); + new NodeSpillConfig(), + new TaskManagerConfig()); factory = new GenericPartitioningSpillerFactory(singleStreamSpillerFactory); scheduledExecutor = newSingleThreadScheduledExecutor(); } diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestLogicalPlanner.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestLogicalPlanner.java index d38c41f4348b..870207ff9311 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/TestLogicalPlanner.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestLogicalPlanner.java @@ -2237,6 +2237,25 @@ public void testExplainAnalyze() node(ExplainAnalyzeNode.class, exchange(LOCAL, GATHER, strictTableScan("nation", ImmutableMap.of("regionkey", "regionkey")))))); + + assertDistributedPlan(""" + EXPLAIN ANALYZE + SELECT * FROM + (SELECT * from nation, region) + UNION ALL + (SELECT * from nation, region) + """, + output( + node(ExplainAnalyzeNode.class, + exchange(LOCAL, GATHER, + exchange(REMOTE, GATHER, + exchange(REMOTE, GATHER, + join(INNER, builder -> builder + .left(tableScan("nation", ImmutableMap.of("regionkey_0", "regionkey"))) + .right(anyTree(tableScan("region", ImmutableMap.of("regionkey_1", "regionkey"))))), + join(INNER, builder -> builder + .left(tableScan("nation", ImmutableMap.of("regionkey_2", "regionkey"))) + .right(anyTree(tableScan("region", ImmutableMap.of("regionkey_3", "regionkey"))))))))))); } @Test diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestMerge.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestMerge.java new file mode 100644 index 000000000000..6e29acc2c801 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestMerge.java @@ -0,0 +1,155 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.Session; +import io.trino.connector.MockConnectorColumnHandle; +import io.trino.connector.MockConnectorFactory; +import io.trino.connector.MockConnectorPlugin; +import io.trino.connector.MockConnectorTableHandle; +import io.trino.metadata.ResolvedFunction; +import io.trino.metadata.TestingFunctionResolution; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.RowType; +import io.trino.spi.type.Type; +import io.trino.sql.ir.Call; +import io.trino.sql.ir.Case; +import io.trino.sql.ir.Cast; +import io.trino.sql.ir.Coalesce; +import io.trino.sql.ir.Constant; +import io.trino.sql.ir.FieldReference; +import io.trino.sql.ir.IsNull; +import io.trino.sql.ir.Reference; +import io.trino.sql.ir.Row; +import io.trino.sql.ir.WhenClause; +import io.trino.sql.planner.assertions.BasePlanTest; +import io.trino.testing.PlanTester; +import org.junit.jupiter.api.Test; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; +import static io.trino.spi.StandardErrorCode.MERGE_TARGET_ROW_MULTIPLE_MATCHES; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.BooleanType.BOOLEAN; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.sql.analyzer.TypeSignatureProvider.fromTypes; +import static io.trino.sql.ir.Booleans.TRUE; +import static io.trino.sql.planner.assertions.PlanMatchPattern.anyTree; +import static io.trino.sql.planner.assertions.PlanMatchPattern.assignUniqueId; +import static io.trino.sql.planner.assertions.PlanMatchPattern.expression; +import static io.trino.sql.planner.assertions.PlanMatchPattern.filter; +import static io.trino.sql.planner.assertions.PlanMatchPattern.join; +import static io.trino.sql.planner.assertions.PlanMatchPattern.markDistinct; +import static io.trino.sql.planner.assertions.PlanMatchPattern.project; +import static io.trino.sql.planner.assertions.PlanMatchPattern.tableScan; +import static io.trino.sql.planner.plan.JoinType.RIGHT; +import static io.trino.testing.TestingSession.testSessionBuilder; + +public class TestMerge + extends BasePlanTest +{ + private static final TestingFunctionResolution FUNCTIONS = new TestingFunctionResolution(); + private static final ResolvedFunction FAIL = FUNCTIONS.resolveFunction("fail", fromTypes(INTEGER, VARCHAR)); + private static final ResolvedFunction NOT = FUNCTIONS.resolveFunction("$not", fromTypes(BOOLEAN)); + private static final Type ROW_TYPE = RowType.anonymousRow(INTEGER, INTEGER, BOOLEAN, TINYINT, INTEGER); + + @Override + protected PlanTester createPlanTester() + { + Session.SessionBuilder sessionBuilder = testSessionBuilder() + .setCatalog("mock") + .setSchema("schema") + .setSystemProperty(ENABLE_DYNAMIC_FILTERING, "false"); + + PlanTester planTester = PlanTester.create(sessionBuilder.build()); + planTester.installPlugin( + new MockConnectorPlugin(MockConnectorFactory.builder() + .withGetTableHandle((session, schemaTableName) -> { + if (schemaTableName.getTableName().equals("test_table_merge_target")) { + return new MockConnectorTableHandle(schemaTableName); + } + + if (schemaTableName.getTableName().equals("test_table_merge_source")) { + return new MockConnectorTableHandle(schemaTableName); + } + + return null; + }) + .withGetColumns(name -> ImmutableList.of( + new ColumnMetadata("column1", INTEGER), + new ColumnMetadata("column2", INTEGER))) + .build())); + planTester.createCatalog("mock", "mock", ImmutableMap.of()); + return planTester; + } + + @Test + public void testMergeWithSimpleSelect() + { + // one join + assertPlan( + "MERGE INTO test_table_merge_target a " + + "USING test_table_merge_source b " + + "ON a.column1 = b.column1 " + + "WHEN MATCHED " + + "THEN UPDATE SET column2 = b.column2 " + + "WHEN NOT MATCHED " + + "THEN INSERT (column1 ,column2) VALUES (b.column1, b.column2)", + anyTree( + filter( + new Case( + ImmutableList.of(new WhenClause(new Call(NOT, ImmutableList.of(new Reference(BOOLEAN, "is_distinct"))), new Cast(new Call(FAIL, ImmutableList.of(new Constant(INTEGER, (long) MERGE_TARGET_ROW_MULTIPLE_MATCHES.toErrorCode().getCode()), new Constant(VARCHAR, utf8Slice("One MERGE target table row matched more than one source row")))), BOOLEAN))), + TRUE), + markDistinct("is_distinct", ImmutableList.of("unique_id", "case_number"), + anyTree( + project(ImmutableMap.of( + "unique_id", expression(new Coalesce(ImmutableList.of(new Reference(BIGINT, "target_unique_id"), new Reference(BIGINT, "source_unique_id")))), + "field", expression(new Reference(BIGINT, "field")), + "merge_row", expression(new Reference(ROW_TYPE, "merge_row")), + "case_number", expression(new FieldReference(new Reference(ROW_TYPE, "merge_row"), 4))), + project(ImmutableMap.of( + "field", expression(new Reference(BIGINT, "field")), + "merge_row", expression(new Case( + ImmutableList.of( + new WhenClause(new Reference(BOOLEAN, "present"), new Row(ImmutableList.of(new Reference(INTEGER, "column1"), new Reference(INTEGER, "column2_1"), new Call(NOT, ImmutableList.of(new IsNull(new Reference(BOOLEAN, "present")))), new Constant(TINYINT, 3L), new Constant(INTEGER, 0L)), ROW_TYPE)), + new WhenClause(new IsNull(new Reference(BOOLEAN, "present")), new Row(ImmutableList.of(new Reference(INTEGER, "column1_0"), new Reference(INTEGER, "column2_1"), new Call(NOT, ImmutableList.of(new IsNull(new Reference(BOOLEAN, "present")))), new Constant(TINYINT, 1L), new Constant(INTEGER, 1L)), ROW_TYPE))), + new Constant(ROW_TYPE, null))), + "target_unique_id", expression(new Reference(BIGINT, "target_unique_id")), + "source_unique_id", expression(new Reference(BIGINT, "source_unique_id"))), + join(RIGHT, builder -> builder + .equiCriteria("column1", "column1_0") + .left( + project(ImmutableMap.of( + "column1", expression(new Reference(INTEGER, "column1")), + "field", expression(new Reference(BIGINT, "field")), + "target_unique_id", expression(new Reference(BIGINT, "target_unique_id")), + "present", expression(new Constant(BOOLEAN, true))), + assignUniqueId("target_unique_id", + tableScan( + tableHandle -> ((MockConnectorTableHandle) tableHandle).getTableName().getTableName().equals("test_table_merge_target"), + TupleDomain.all(), + ImmutableMap.of( + "column1", columnHandle -> ((MockConnectorColumnHandle) columnHandle).name().equals("column1"), + "field", columnHandle -> ((MockConnectorColumnHandle) columnHandle).name().equals("merge_row_id")))))) + .right( + anyTree( + assignUniqueId("source_unique_id", + tableScan("test_table_merge_source", ImmutableMap.of("column1_0", "column1", "column2_1", "column2"))))))))))))); + } +} diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/assertions/ExpressionMatcher.java b/core/trino-main/src/test/java/io/trino/sql/planner/assertions/ExpressionMatcher.java index 5ee85817ae78..47882374bde7 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/assertions/ExpressionMatcher.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/assertions/ExpressionMatcher.java @@ -69,7 +69,7 @@ public Optional getAssignedSymbol(PlanNode node, Session session, Metada private static Map getAssignments(PlanNode node) { if (node instanceof ProjectNode projectNode) { - return projectNode.getAssignments().getMap(); + return projectNode.getAssignments().assignments(); } return null; } diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/assertions/StrictAssignedSymbolsMatcher.java b/core/trino-main/src/test/java/io/trino/sql/planner/assertions/StrictAssignedSymbolsMatcher.java index 45369671341c..63c2349c2e73 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/assertions/StrictAssignedSymbolsMatcher.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/assertions/StrictAssignedSymbolsMatcher.java @@ -58,7 +58,7 @@ protected Set getExpectedSymbols(PlanNode node, Session session, Metadat public static Function> actualAssignments() { - return node -> ((ProjectNode) node).getAssignments().getSymbols(); + return node -> ((ProjectNode) node).getAssignments().outputs(); } public static Function> actualSubqueryAssignments() diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/assertions/SymbolAliases.java b/core/trino-main/src/test/java/io/trino/sql/planner/assertions/SymbolAliases.java index f7422481007d..9419c754888e 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/assertions/SymbolAliases.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/assertions/SymbolAliases.java @@ -101,7 +101,7 @@ public Optional getOptional(String alias) private Map getUpdatedAssignments(Assignments assignments) { ImmutableMap.Builder mapUpdate = ImmutableMap.builder(); - for (Map.Entry assignment : assignments.getMap().entrySet()) { + for (Map.Entry assignment : assignments.assignments().entrySet()) { for (Map.Entry existingAlias : map.entrySet()) { if (assignment.getValue().equals(existingAlias.getValue())) { // Simple symbol rename diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/test/RuleAssert.java b/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/test/RuleAssert.java index 3214ae4a6971..2c8b54e80688 100644 --- a/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/test/RuleAssert.java +++ b/core/trino-main/src/test/java/io/trino/sql/planner/iterative/rule/test/RuleAssert.java @@ -94,7 +94,7 @@ public void doesNotFire() """, rule, textLogicalPlan(plan, planTester.getPlannerContext().getMetadata(), planTester.getPlannerContext().getFunctionManager(), StatsAndCosts.empty(), session, 2, false), - textLogicalPlan(ruleApplication.result.getTransformedPlan().get(), planTester.getPlannerContext().getMetadata(), planTester.getPlannerContext().getFunctionManager(), StatsAndCosts.empty(), session, 2, false))); + textLogicalPlan(ruleApplication.result.transformedPlan().get(), planTester.getPlannerContext().getMetadata(), planTester.getPlannerContext().getFunctionManager(), StatsAndCosts.empty(), session, 2, false))); } } finally { @@ -253,7 +253,7 @@ private boolean wasRuleApplied() public PlanNode getTransformedPlan() { - return result.getTransformedPlan().orElseThrow(() -> new IllegalStateException("Rule did not produce transformed plan")); + return result.transformedPlan().orElseThrow(() -> new IllegalStateException("Rule did not produce transformed plan")); } } } diff --git a/core/trino-main/src/test/java/io/trino/sql/query/TestDropCatalog.java b/core/trino-main/src/test/java/io/trino/sql/query/TestDropCatalog.java new file mode 100644 index 000000000000..b2a2dae3416c --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/query/TestDropCatalog.java @@ -0,0 +1,160 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.query; + +import io.airlift.units.Duration; +import io.trino.client.NodeVersion; +import io.trino.connector.MockConnectorFactory; +import io.trino.connector.MockConnectorPlugin; +import io.trino.connector.MockConnectorTableHandle; +import io.trino.execution.QueryStateMachine; +import io.trino.execution.warnings.WarningCollector; +import io.trino.spi.TrinoException; +import io.trino.spi.resourcegroups.ResourceGroupId; +import io.trino.testing.QueryRunner; +import io.trino.testing.StandaloneQueryRunner; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; + +import java.net.URI; +import java.util.Optional; + +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static io.trino.SessionTestUtils.TEST_SESSION; +import static io.trino.execution.querystats.PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector; +import static io.trino.spi.StandardErrorCode.NOT_FOUND; +import static io.trino.spi.session.PropertyMetadata.stringProperty; +import static io.trino.testing.TestingSession.testSession; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT; + +@TestInstance(PER_CLASS) +@Execution(CONCURRENT) +public class TestDropCatalog +{ + private QueryAssertions queryAssertions; + + @BeforeAll + public void setUp() + { + Duration catalogPruneInterval = new Duration(1, SECONDS); // lowest allowed + QueryRunner queryRunner = new StandaloneQueryRunner(TEST_SESSION, + server -> server.addProperty("catalog.prune.update-interval", catalogPruneInterval.toString())); + queryRunner.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder() + .withName("connector_with_cleanup_query") + .withSessionProperty(stringProperty( + "baz", + "test property", + null, + false)) + .withGetTableHandle((_, name) -> switch (name.toString()) { + case "default.existing" -> new MockConnectorTableHandle(name); + default -> { + throw new TrinoException(NOT_FOUND, "Table not found: " + name); + } + }) + .withCleanupQuery(session -> { + // Increase chances of a bad interleaving of threads for the test to be rather deterministic + try { + Thread.sleep(catalogPruneInterval.toMillis()); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + // simulate cleanupQuery that checks session state + session.getProperty("baz", String.class); + }) + .build())); + this.queryAssertions = new QueryAssertions(queryRunner); + } + + @AfterAll + public void tearDown() + { + if (queryAssertions != null) { + queryAssertions.close(); + } + queryAssertions = null; + } + + @Test + void testDropCatalogWithCleanupQuery() + { + QueryRunner queryRunner = queryAssertions.getQueryRunner(); + String catalogName = "catalog_with_cleanup_query"; + queryRunner.createCatalog(catalogName, "connector_with_cleanup_query"); + assertCatalogExists(catalogName); + + queryRunner.execute("DROP CATALOG " + catalogName); + + assertCatalogDoesNotExist(catalogName); + } + + private void assertCatalogExists(String catalogName) + { + QueryRunner queryRunner = queryAssertions.getQueryRunner(); + + QueryStateMachine fakeQuery = createNewQuery(); + assertThat(queryRunner.getPlannerContext().getMetadata().catalogExists(fakeQuery.getSession(), catalogName)).isTrue(); + + assertThat(queryAssertions.query("SHOW SCHEMAS FROM " + catalogName)) + .containsAll("VALUES VARCHAR 'information_schema'"); + + assertThat(queryAssertions.query("SELECT * FROM " + catalogName + ".default.existing")) + .returnsEmptyResult(); + } + + private void assertCatalogDoesNotExist(String catalogName) + { + QueryRunner queryRunner = queryAssertions.getQueryRunner(); + + QueryStateMachine fakeQuery = createNewQuery(); + assertThat(queryRunner.getPlannerContext().getMetadata().catalogExists(fakeQuery.getSession(), catalogName)).isFalse(); + + assertThat(queryAssertions.query("SHOW SCHEMAS FROM " + catalogName)) + .failure().hasMessage("Catalog '%s' not found".formatted(catalogName)); + + assertThat(queryAssertions.query("SELECT * FROM " + catalogName + ".default.existing")) + .failure().hasMessageContaining("Catalog '%s' not found".formatted(catalogName)); + } + + private QueryStateMachine createNewQuery() + { + QueryRunner queryRunner = queryAssertions.getQueryRunner(); + return QueryStateMachine.begin( + Optional.empty(), + "test", + Optional.empty(), + testSession(queryRunner.getDefaultSession()), + URI.create("fake://uri"), + new ResourceGroupId("test"), + false, + queryRunner.getTransactionManager(), + queryRunner.getAccessControl(), + directExecutor(), + queryRunner.getPlannerContext().getMetadata(), + WarningCollector.NOOP, + createPlanOptimizersStatsCollector(), + Optional.empty(), + true, + Optional.empty(), + new NodeVersion("test")); + } +} diff --git a/core/trino-main/src/test/java/io/trino/sql/query/TestRowPatternMatching.java b/core/trino-main/src/test/java/io/trino/sql/query/TestRowPatternMatching.java index e0618766aa84..9121dfd51eab 100644 --- a/core/trino-main/src/test/java/io/trino/sql/query/TestRowPatternMatching.java +++ b/core/trino-main/src/test/java/io/trino/sql/query/TestRowPatternMatching.java @@ -1020,8 +1020,8 @@ public void testNavigationFunctions() assertThat(assertions.query(format(query, "FIRST(value, 2)"))) .matches("VALUES " + - " (1, 30), " + - " (2, 30), " + + " (1, null), " + + " (2, null), " + " (3, 30) "); assertThat(assertions.query(format(query, "LAST(value, 10)"))) diff --git a/core/trino-main/src/test/java/io/trino/type/AbstractTestType.java b/core/trino-main/src/test/java/io/trino/type/AbstractTestType.java index 220541ee5f97..77d639cc9eb6 100644 --- a/core/trino-main/src/test/java/io/trino/type/AbstractTestType.java +++ b/core/trino-main/src/test/java/io/trino/type/AbstractTestType.java @@ -328,7 +328,7 @@ protected void assertPositionEquals(ValueBlock block, int position, Object expec assertPositionValue(block.getRegion(position, block.getPositionCount() - position), 0, expectedStackValue, hash, expectedObjectValue); BlockBuilder blockBuilder = type.createBlockBuilder(null, 1); - type.appendTo(block, position, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); assertPositionValue(blockBuilder.buildValueBlock(), 0, expectedStackValue, hash, expectedObjectValue); if (expectedStackValue != null) { diff --git a/core/trino-main/src/test/java/io/trino/type/TestBigintArrayType.java b/core/trino-main/src/test/java/io/trino/type/TestBigintArrayType.java index 88d1e29d1b57..9a8b192fde29 100644 --- a/core/trino-main/src/test/java/io/trino/type/TestBigintArrayType.java +++ b/core/trino-main/src/test/java/io/trino/type/TestBigintArrayType.java @@ -51,7 +51,7 @@ protected Object getGreaterValue(Object value) Block block = (Block) value; BlockBuilder blockBuilder = BIGINT.createFixedSizeBlockBuilder(block.getPositionCount() + 1); for (int i = 0; i < block.getPositionCount(); i++) { - BIGINT.appendTo(block, i, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(i)); } BIGINT.writeLong(blockBuilder, 1L); diff --git a/core/trino-main/src/test/java/io/trino/type/TestIntegerArrayType.java b/core/trino-main/src/test/java/io/trino/type/TestIntegerArrayType.java index 8d069dcfc78e..497e99735150 100644 --- a/core/trino-main/src/test/java/io/trino/type/TestIntegerArrayType.java +++ b/core/trino-main/src/test/java/io/trino/type/TestIntegerArrayType.java @@ -51,7 +51,7 @@ protected Object getGreaterValue(Object value) Block block = (Block) value; BlockBuilder blockBuilder = INTEGER.createFixedSizeBlockBuilder(block.getPositionCount() + 1); for (int i = 0; i < block.getPositionCount(); i++) { - INTEGER.appendTo(block, i, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(i)); } INTEGER.writeLong(blockBuilder, 1L); diff --git a/core/trino-main/src/test/java/io/trino/type/TestSmallintArrayType.java b/core/trino-main/src/test/java/io/trino/type/TestSmallintArrayType.java index a338c93aaf23..e84fda120dff 100644 --- a/core/trino-main/src/test/java/io/trino/type/TestSmallintArrayType.java +++ b/core/trino-main/src/test/java/io/trino/type/TestSmallintArrayType.java @@ -51,7 +51,7 @@ protected Object getGreaterValue(Object value) Block block = (Block) value; BlockBuilder blockBuilder = SMALLINT.createFixedSizeBlockBuilder(block.getPositionCount() + 1); for (int i = 0; i < block.getPositionCount(); i++) { - SMALLINT.appendTo(block, i, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(i)); } SMALLINT.writeLong(blockBuilder, 1L); diff --git a/core/trino-main/src/test/java/io/trino/type/TestTinyintArrayType.java b/core/trino-main/src/test/java/io/trino/type/TestTinyintArrayType.java index 0d740b33a80f..b25ee0913bab 100644 --- a/core/trino-main/src/test/java/io/trino/type/TestTinyintArrayType.java +++ b/core/trino-main/src/test/java/io/trino/type/TestTinyintArrayType.java @@ -51,7 +51,7 @@ protected Object getGreaterValue(Object value) Block block = (Block) value; BlockBuilder blockBuilder = TINYINT.createFixedSizeBlockBuilder(block.getPositionCount() + 1); for (int i = 0; i < block.getPositionCount(); i++) { - TINYINT.appendTo(block, i, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(i)); } TINYINT.writeLong(blockBuilder, 1L); diff --git a/core/trino-main/src/test/java/io/trino/type/TestVarcharArrayType.java b/core/trino-main/src/test/java/io/trino/type/TestVarcharArrayType.java index 0015ab005c09..fa1344d33871 100644 --- a/core/trino-main/src/test/java/io/trino/type/TestVarcharArrayType.java +++ b/core/trino-main/src/test/java/io/trino/type/TestVarcharArrayType.java @@ -51,7 +51,7 @@ protected Object getGreaterValue(Object value) Block block = (Block) value; BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, block.getPositionCount() + 1); for (int i = 0; i < block.getPositionCount(); i++) { - VARCHAR.appendTo(block, i, blockBuilder); + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(i)); } VARCHAR.writeSlice(blockBuilder, utf8Slice("_")); diff --git a/core/trino-parser/pom.xml b/core/trino-parser/pom.xml index e7738c80fd4b..d68a6d930d85 100644 --- a/core/trino-parser/pom.xml +++ b/core/trino-parser/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/core/trino-server-core/pom.xml b/core/trino-server-core/pom.xml index 5be5633201a6..ec0fb29738f2 100644 --- a/core/trino-server-core/pom.xml +++ b/core/trino-server-core/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/core/trino-server-main/pom.xml b/core/trino-server-main/pom.xml index abca005a21e8..6f33bfc568fa 100644 --- a/core/trino-server-main/pom.xml +++ b/core/trino-server-main/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/core/trino-server/pom.xml b/core/trino-server/pom.xml index e493d32dc81d..5817e02d2081 100644 --- a/core/trino-server/pom.xml +++ b/core/trino-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/core/trino-spi/pom.xml b/core/trino-spi/pom.xml index 45f6e7fa2697..09396d9b9930 100644 --- a/core/trino-spi/pom.xml +++ b/core/trino-spi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -216,621 +216,126 @@ - - java.method.removed - method io.trino.spi.Page io.trino.spi.Page::getLoadedPage(int[], int[]) - - - java.method.removed - method io.trino.spi.block.ArrayBlock io.trino.spi.block.ArrayBlock::getLoadedBlock() - - - java.method.removed - method boolean io.trino.spi.block.ArrayBlock::isLoaded() - - - java.method.removed - method io.trino.spi.block.Block io.trino.spi.block.Block::getLoadedBlock() - - - java.method.removed - method boolean io.trino.spi.block.Block::isLoaded() - - - java.method.removed - method io.trino.spi.block.Block io.trino.spi.block.DictionaryBlock::getLoadedBlock() - - - java.method.removed - method boolean io.trino.spi.block.DictionaryBlock::isLoaded() - - - java.class.removed - class io.trino.spi.block.LazyBlock - - - java.class.removed - class io.trino.spi.block.LazyBlockEncoding - - - java.class.removed - interface io.trino.spi.block.LazyBlockLoader - - - java.method.removed - method io.trino.spi.block.Block io.trino.spi.block.MapBlock::getLoadedBlock() - - - java.method.removed - method boolean io.trino.spi.block.MapBlock::isLoaded() - - - java.method.removed - method io.trino.spi.block.Block io.trino.spi.block.RowBlock::getLoadedBlock() - - - java.method.removed - method boolean io.trino.spi.block.RowBlock::isLoaded() - - - java.method.removed - method io.trino.spi.block.Block io.trino.spi.block.RunLengthEncodedBlock::getLoadedBlock() - - - java.method.removed - method boolean io.trino.spi.block.RunLengthEncodedBlock::isLoaded() - - - java.method.removed - method io.trino.spi.Page io.trino.spi.Page::getLoadedPage(int) - - - java.method.removed - method io.trino.spi.Page io.trino.spi.Page::getLoadedPage() - - - java.method.removed - method io.trino.spi.Page io.trino.spi.Page::getLoadedPage(int[]) - - - true - java.method.returnTypeTypeParametersChanged - method java.util.List<java.lang.String> io.trino.spi.eventlistener.QueryInputMetadata::getColumns() - - - true - java.method.parameterTypeParameterChanged - method void io.trino.spi.eventlistener.QueryInputMetadata::<init>(java.lang.String, io.trino.spi.connector.CatalogHandle.CatalogVersion, java.lang.String, java.lang.String, java.util.List<java.lang.String>, java.util.Optional<java.lang.Object>, io.trino.spi.metrics.Metrics, java.util.OptionalLong, java.util.OptionalLong) - - - true - java.method.numberOfParametersChanged - method java.util.Optional<java.lang.Object> io.trino.spi.connector.ConnectorMetadata::getInfo(io.trino.spi.connector.ConnectorTableHandle) - - - true - java.method.numberOfParametersChanged - method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Optional<java.lang.String>) - - - true - java.method.returnTypeChanged - method io.trino.spi.eventlistener.DoubleSymmetricDistribution io.trino.spi.eventlistener.StageTaskStatistics::getGetSplitDistribution() - - - true - java.method.parameterTypeChanged - method void io.trino.spi.eventlistener.StageTaskStatistics::<init>(int, int, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongDistribution, io.trino.spi.eventlistener.LongSymmetricDistribution, io.trino.spi.eventlistener.LongSymmetricDistribution, io.trino.spi.eventlistener.LongSymmetricDistribution, io.trino.spi.eventlistener.LongSymmetricDistribution, io.trino.spi.eventlistener.LongSymmetricDistribution, io.trino.spi.eventlistener.LongSymmetricDistribution, io.trino.spi.eventlistener.DoubleSymmetricDistribution, io.trino.spi.eventlistener.DoubleSymmetricDistribution, io.trino.spi.eventlistener.DoubleSymmetricDistribution, io.trino.spi.eventlistener.DoubleSymmetricDistribution, io.trino.spi.eventlistener.DoubleSymmetricDistribution, io.trino.spi.eventlistener.DoubleSymmetricDistribution, io.trino.spi.eventlistener.DoubleSymmetricDistribution) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.ArrayBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.ArrayBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.Block::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.Block::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.ByteArrayBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.ByteArrayBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.DictionaryBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.DictionaryBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.Fixed12Block::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.Fixed12Block::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.Int128ArrayBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.Int128ArrayBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.IntArrayBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.IntArrayBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.LongArrayBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.LongArrayBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.MapBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.MapBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.RowBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.RowBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.RunLengthEncodedBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.RunLengthEncodedBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.ShortArrayBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.ShortArrayBlock::getPositionsSizeInBytes(boolean[], int) - - - java.method.removed - method java.util.OptionalInt io.trino.spi.block.VariableWidthBlock::fixedSizeInBytesPerPosition() - - - java.method.removed - method long io.trino.spi.block.VariableWidthBlock::getPositionsSizeInBytes(boolean[], int) - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.ArrayType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.ArrayType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.BigintType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.BigintType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.BooleanType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.BooleanType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.CharType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.CharType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.DateType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.DateType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.DoubleType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.DoubleType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.HyperLogLogType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.HyperLogLogType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.IntegerType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.IntegerType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.MapType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.MapType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.P4HyperLogLogType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.P4HyperLogLogType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.QuantileDigestType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.QuantileDigestType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.RealType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.RealType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.RowType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.RowType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.SmallintType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.SmallintType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.TimeType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.TimeType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.TinyintType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.TinyintType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.Type::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.Type::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.UuidType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.UuidType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.VarbinaryType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.VarbinaryType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.numberOfParametersChanged - method java.lang.Object io.trino.spi.type.VarcharType::getObjectValue(io.trino.spi.connector.ConnectorSession, io.trino.spi.block.Block, int) - method java.lang.Object io.trino.spi.type.VarcharType::getObjectValue(io.trino.spi.block.Block, int) - Type cleanup - - - true - java.method.parameterTypeChanged - parameter java.lang.String io.trino.spi.predicate.AllOrNoneValueSet::toString(===io.trino.spi.connector.ConnectorSession===) - parameter java.lang.String io.trino.spi.predicate.AllOrNoneValueSet::toString(===int===) - 0 - Type cleanup - - - true - java.method.removed - method java.lang.String io.trino.spi.predicate.AllOrNoneValueSet::toString(io.trino.spi.connector.ConnectorSession, int) - Type cleanup - - - true - java.method.parameterTypeChanged - parameter java.lang.String io.trino.spi.predicate.Domain::toString(===io.trino.spi.connector.ConnectorSession===) - parameter java.lang.String io.trino.spi.predicate.Domain::toString(===int===) - 0 - Type cleanup - - - true - java.method.removed - method java.lang.String io.trino.spi.predicate.Domain::toString(io.trino.spi.connector.ConnectorSession, int) - Type cleanup - - - true - java.method.parameterTypeChanged - parameter java.lang.String io.trino.spi.predicate.EquatableValueSet::toString(===io.trino.spi.connector.ConnectorSession===) - parameter java.lang.String io.trino.spi.predicate.EquatableValueSet::toString(===int===) - 0 - Type cleanup - - - true - java.method.removed - method java.lang.String io.trino.spi.predicate.EquatableValueSet::toString(io.trino.spi.connector.ConnectorSession, int) - Type cleanup - - - true - java.method.removed - method java.lang.String io.trino.spi.predicate.Range::toString(io.trino.spi.connector.ConnectorSession) - Type cleanup - true - java.method.parameterTypeChanged - parameter java.lang.String io.trino.spi.predicate.SortedRangeSet::toString(===io.trino.spi.connector.ConnectorSession===) - parameter java.lang.String io.trino.spi.predicate.SortedRangeSet::toString(===int===) - 0 - Type cleanup - - - true - java.method.removed - method java.lang.String io.trino.spi.predicate.SortedRangeSet::toString(io.trino.spi.connector.ConnectorSession, int) - Type cleanup - - - true - java.method.removed - method java.lang.String io.trino.spi.predicate.TupleDomain<T>::toString(io.trino.spi.connector.ConnectorSession) - Type cleanup - - - true - java.method.parameterTypeChanged - parameter java.lang.String io.trino.spi.predicate.ValueSet::toString(===io.trino.spi.connector.ConnectorSession===) - parameter java.lang.String io.trino.spi.predicate.ValueSet::toString(===int===) - 0 - Type cleanup - - - true - java.method.removed - method java.lang.String io.trino.spi.predicate.ValueSet::toString(io.trino.spi.connector.ConnectorSession, int) - Type cleanup + java.method.noLongerDefault + method void io.trino.spi.connector.Connector::shutdown() + method void io.trino.spi.connector.Connector::shutdown() + Require connector to implement shutdown to prevent leaks true - java.method.removed - method java.lang.String io.trino.spi.NodeManager::getEnvironment() - Unused - - - java.class.removed - @interface io.trino.spi.Experimental + java.method.nowAbstract + method void io.trino.spi.connector.Connector::shutdown() + method void io.trino.spi.connector.Connector::shutdown() + Require connector to implement shutdown to prevent leaks java.annotation.removed - io.trino.spi.Experimental + method java.lang.String io.trino.spi.QueryId::toString() + method java.lang.String io.trino.spi.QueryId::toString() + @com.fasterxml.jackson.annotation.JsonValue + QueryId converted to a record true - java.method.removed - method io.trino.spi.Page io.trino.spi.connector.EmptyPageSource::getNextPage() - Deprecated + java.class.kindChanged + class io.trino.spi.QueryId + class io.trino.spi.QueryId + QueryId converted to a record true java.method.removed - method io.trino.spi.Page io.trino.spi.connector.FixedPageSource::getNextPage() - Deprecated + method void io.trino.spi.resourcegroups.SelectionCriteria::<init>(boolean, java.lang.String, java.util.Set<java.lang.String>, java.util.Optional<java.lang.String>, java.util.Set<java.lang.String>, io.trino.spi.session.ResourceEstimates, java.util.Optional<java.lang.String>) + Remove a deprecated constructor true - java.method.removed - method io.trino.spi.Page io.trino.spi.connector.RecordPageSource::getNextPage() - Deprecated + java.method.varargOverloadsOnlyDifferInVarargParameter + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, boolean[]) + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, boolean[]) + Revapi now detects new API changes to vararg args true - java.method.addedToInterface - method long io.trino.spi.resourcegroups.ResourceGroup::getHardPhysicalDataScanLimitBytes() - Add physical data scan tracking to resource groups + java.method.varargOverloadsOnlyDifferInVarargParameter + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, double[]) + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, double[]) + Revapi now detects new API changes to vararg args true - java.method.addedToInterface - method void io.trino.spi.resourcegroups.ResourceGroup::setHardPhysicalDataScanLimitBytes(long) - Add physical data scan tracking to resource groups + java.method.varargOverloadsOnlyDifferInVarargParameter + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, java.lang.String[]) + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, java.lang.String[]) + Revapi now detects new API changes to vararg args true - java.method.addedToInterface - method long io.trino.spi.resourcegroups.ResourceGroup::getPhysicalDataScanQuotaGenerationBytesPerSecond() - Add physical data scan tracking to resource groups + java.method.varargOverloadsOnlyDifferInVarargParameter + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, long[]) + method io.opentelemetry.api.common.AttributesBuilder io.opentelemetry.api.common.AttributesBuilder::put(java.lang.String, long[]) + Revapi now detects new API changes to vararg args true - java.method.addedToInterface - method void io.trino.spi.resourcegroups.ResourceGroup::setPhysicalDataScanQuotaGenerationBytesPerSecond(long) - Add physical data scan tracking to resource groups + java.method.varargOverloadsOnlyDifferInVarargParameter + method io.opentelemetry.api.common.Value<java.util.List<io.opentelemetry.api.common.KeyValue>> io.opentelemetry.api.common.Value<T>::of(io.opentelemetry.api.common.KeyValue[]) + method io.opentelemetry.api.common.Value<java.util.List<io.opentelemetry.api.common.KeyValue>> io.opentelemetry.api.common.Value<T>::of(io.opentelemetry.api.common.KeyValue[]) + Revapi now detects new API changes to vararg args true - java.method.visibilityIncreased - method void io.trino.spi.type.Int128Math::multiply(long, long, long, long, long[], int) - method void io.trino.spi.type.Int128Math::multiply(long, long, long, long, long[], int) - private - public + java.method.varargOverloadsOnlyDifferInVarargParameter + method io.opentelemetry.api.common.Value<java.util.List<io.opentelemetry.api.common.Value<?>>> io.opentelemetry.api.common.Value<T>::of(io.opentelemetry.api.common.Value<?>[]) + method io.opentelemetry.api.common.Value<java.util.List<io.opentelemetry.api.common.Value<?>>> io.opentelemetry.api.common.Value<T>::of(io.opentelemetry.api.common.Value<?>[]) + Revapi now detects new API changes to vararg args true java.method.visibilityIncreased - method long io.trino.spi.type.Int128Math::signExtension(long) - method long io.trino.spi.type.Int128Math::signExtension(long) - private + method boolean[] io.trino.spi.block.RowBlock::getRawRowIsNull() + method boolean[] io.trino.spi.block.RowBlock::getRawRowIsNull() + package public + Allow direct access to isNull mask on RowBlock for performance critical sections - true - java.method.numberOfParametersChanged - method io.trino.spi.connector.ConnectorInsertTableHandle io.trino.spi.connector.ConnectorMetadata::beginRefreshMaterializedView(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableHandle, java.util.List<io.trino.spi.connector.ConnectorTableHandle>, io.trino.spi.connector.RetryMode, io.trino.spi.RefreshType) - method io.trino.spi.connector.ConnectorInsertTableHandle io.trino.spi.connector.ConnectorMetadata::beginRefreshMaterializedView(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableHandle, java.util.List<io.trino.spi.connector.ConnectorTableHandle>, boolean, io.trino.spi.connector.RetryMode, io.trino.spi.RefreshType) - - - true - java.method.numberOfParametersChanged - method java.util.Optional<io.trino.spi.connector.ConnectorOutputMetadata> io.trino.spi.connector.ConnectorMetadata::finishRefreshMaterializedView(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableHandle, io.trino.spi.connector.ConnectorInsertTableHandle, java.util.Collection<io.airlift.slice.Slice>, java.util.Collection<io.trino.spi.statistics.ComputedStatistics>, java.util.List<io.trino.spi.connector.ConnectorTableHandle>, java.util.List<java.lang.String>) - method java.util.Optional<io.trino.spi.connector.ConnectorOutputMetadata> io.trino.spi.connector.ConnectorMetadata::finishRefreshMaterializedView(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableHandle, io.trino.spi.connector.ConnectorInsertTableHandle, java.util.Collection<io.airlift.slice.Slice>, java.util.Collection<io.trino.spi.statistics.ComputedStatistics>, java.util.List<io.trino.spi.connector.ConnectorTableHandle>, boolean, boolean) - - - true - java.method.numberOfParametersChanged - method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.List<io.trino.spi.eventlistener.DynamicFilterDomainStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Optional<java.lang.String>) - method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.List<io.trino.spi.eventlistener.DynamicFilterDomainStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Optional<java.lang.String>) - - - true - java.method.removed - method long io.trino.spi.eventlistener.QueryStatistics::getTotalRows() - Redundant with QueryStatistics::getProcessedInputRows() + java.method.returnTypeChanged + method void io.trino.spi.connector.ConnectorMetadata::executeTableExecute(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableExecuteHandle) + method java.util.Map<java.lang.String, java.lang.Long> io.trino.spi.connector.ConnectorMetadata::executeTableExecute(io.trino.spi.connector.ConnectorSession, io.trino.spi.connector.ConnectorTableExecuteHandle) - true java.method.removed - method long io.trino.spi.eventlistener.QueryStatistics::getTotalBytes() - Redundant with QueryStatistics::getInternalNetworkBytes() + QueryStatistics::getPhysicalInputBytes() + method int io.trino.spi.PageSorter::decodePageIndex(long) java.method.removed - method io.trino.spi.connector.CatalogHandle io.trino.spi.connector.ConnectorContext::getCatalogHandle() - - - java.method.parameterTypeChanged - parameter io.trino.spi.connector.CatalogHandle io.trino.spi.connector.CatalogHandle::createRootCatalogHandle(io.trino.spi.catalog.CatalogName, ===io.trino.spi.connector.CatalogHandle.CatalogVersion===) - parameter io.trino.spi.connector.CatalogHandle io.trino.spi.connector.CatalogHandle::createRootCatalogHandle(io.trino.spi.catalog.CatalogName, ===io.trino.spi.connector.CatalogVersion===) - 1 - - - java.method.returnTypeChanged - method io.trino.spi.connector.CatalogHandle.CatalogVersion io.trino.spi.connector.CatalogHandle::getVersion() - method io.trino.spi.connector.CatalogVersion io.trino.spi.connector.CatalogHandle::getVersion() - - - java.method.parameterTypeChanged - parameter void io.trino.spi.eventlistener.QueryInputMetadata::<init>(java.util.Optional<java.lang.String>, java.lang.String, ===io.trino.spi.connector.CatalogHandle.CatalogVersion===, java.lang.String, java.lang.String, java.util.List<io.trino.spi.eventlistener.QueryInputMetadata.Column>, java.util.Optional<java.lang.Object>, io.trino.spi.metrics.Metrics, java.util.OptionalLong, java.util.OptionalLong) - parameter void io.trino.spi.eventlistener.QueryInputMetadata::<init>(java.util.Optional<java.lang.String>, java.lang.String, ===io.trino.spi.connector.CatalogVersion===, java.lang.String, java.lang.String, java.util.List<io.trino.spi.eventlistener.QueryInputMetadata.Column>, java.util.Optional<java.lang.Object>, io.trino.spi.metrics.Metrics, java.util.OptionalLong, java.util.OptionalLong) - 2 - - - java.method.returnTypeChanged - method io.trino.spi.connector.CatalogHandle.CatalogVersion io.trino.spi.eventlistener.QueryInputMetadata::getCatalogVersion() - method io.trino.spi.connector.CatalogVersion io.trino.spi.eventlistener.QueryInputMetadata::getCatalogVersion() - - - java.method.parameterTypeChanged - parameter void io.trino.spi.eventlistener.QueryOutputMetadata::<init>(java.lang.String, ===io.trino.spi.connector.CatalogHandle.CatalogVersion===, java.lang.String, java.lang.String, java.util.Optional<java.util.List<io.trino.spi.eventlistener.OutputColumnMetadata>>, java.util.Optional<java.lang.String>, java.util.Optional<java.lang.Boolean>) - parameter void io.trino.spi.eventlistener.QueryOutputMetadata::<init>(java.lang.String, ===io.trino.spi.connector.CatalogVersion===, java.lang.String, java.lang.String, java.util.Optional<java.util.List<io.trino.spi.eventlistener.OutputColumnMetadata>>, java.util.Optional<java.lang.String>, java.util.Optional<java.lang.Boolean>) - 1 + method int io.trino.spi.PageSorter::decodePositionIndex(long) java.method.returnTypeChanged - method io.trino.spi.connector.CatalogHandle.CatalogVersion io.trino.spi.eventlistener.QueryOutputMetadata::getCatalogVersion() - method io.trino.spi.connector.CatalogVersion io.trino.spi.eventlistener.QueryOutputMetadata::getCatalogVersion() - - - java.method.numberOfParametersChanged - method void io.trino.spi.catalog.CatalogProperties::<init>(io.trino.spi.connector.CatalogHandle, io.trino.spi.connector.ConnectorName, java.util.Map<java.lang.String, java.lang.String>) - method void io.trino.spi.catalog.CatalogProperties::<init>(io.trino.spi.catalog.CatalogName, io.trino.spi.connector.CatalogVersion, io.trino.spi.connector.ConnectorName, java.util.Map<java.lang.String, java.lang.String>) - - - java.method.removed - method io.trino.connector.CatalogHandle io.trino.spi.catalog.CatalogProperties::catalogHandle() - - - java.class.removed - io.trino.spi.connector.CatalogHandle + method long[] io.trino.spi.PageSorter::sort(java.util.List<io.trino.spi.type.Type>, java.util.List<io.trino.spi.Page>, java.util.List<java.lang.Integer>, java.util.List<io.trino.spi.connector.SortOrder>, int) + method java.util.Iterator<io.trino.spi.Page> io.trino.spi.PageSorter::sort(java.util.List<io.trino.spi.type.Type>, java.util.List<io.trino.spi.Page>, java.util.List<java.lang.Integer>, java.util.List<io.trino.spi.connector.SortOrder>, int) java.method.removed - method io.trino.spi.connector.CatalogHandle io.trino.spi.catalog.CatalogProperties::catalogHandle() + method double io.trino.spi.metrics.Distribution<T>::getPercentile(double) - true - java.method.visibilityIncreased - method io.trino.spi.block.Block[] io.trino.spi.block.RowBlock::getRawFieldBlocks() - method io.trino.spi.block.Block[] io.trino.spi.block.RowBlock::getRawFieldBlocks() - package - public - getRawFieldBlocks need to be accessed from trino-main - - - true - java.method.removed - method io.trino.spi.block.Block io.trino.spi.block.RowBlock::getFieldBlock(int) + java.method.addedToInterface + method double[] io.trino.spi.metrics.Distribution<T>::getPercentiles(double[]) - true - java.method.noLongerDefault - method void io.trino.spi.connector.Connector::shutdown() - method void io.trino.spi.connector.Connector::shutdown() - Require connector to implement shutdown to prevent leaks + java.method.numberOfParametersChanged + method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.List<io.trino.spi.eventlistener.DynamicFilterDomainStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Optional<java.lang.String>) + method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.List<io.trino.spi.eventlistener.DynamicFilterDomainStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Map<java.lang.String, io.trino.spi.metrics.Metrics>, java.util.Optional<java.lang.String>) - true - java.method.nowAbstract - method void io.trino.spi.connector.Connector::shutdown() - method void io.trino.spi.connector.Connector::shutdown() - Require connector to implement shutdown to prevent leaks + java.method.numberOfParametersChanged + method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.List<io.trino.spi.eventlistener.DynamicFilterDomainStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Map<java.lang.String, io.trino.spi.metrics.Metrics>, java.util.Optional<java.lang.String>) + method void io.trino.spi.eventlistener.QueryStatistics::<init>(java.time.Duration, java.time.Duration, java.time.Duration, java.time.Duration, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, java.util.Optional<java.time.Duration>, long, long, long, long, long, long, long, long, long, long, long, long, long, long, double, double, java.util.List<io.trino.spi.eventlistener.StageGcStatistics>, int, boolean, java.util.List<io.trino.spi.eventlistener.StageCpuDistribution>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferUtilization>, java.util.List<io.trino.spi.eventlistener.StageOutputBufferMetrics>, java.util.List<io.trino.spi.eventlistener.StageTaskStatistics>, java.util.List<io.trino.spi.eventlistener.DynamicFilterDomainStatistics>, java.util.function.Supplier<java.util.List<java.lang.String>>, java.util.List<io.trino.spi.eventlistener.QueryPlanOptimizerStatistics>, java.util.Map<java.lang.String, io.trino.spi.metrics.Metrics>, java.util.Optional<java.lang.String>) @@ -840,7 +345,7 @@ org.revapi revapi-java - 0.28.1 + 0.28.4 diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/NodeVersion.java b/core/trino-spi/src/main/java/io/trino/spi/NodeVersion.java similarity index 96% rename from plugin/trino-hive/src/main/java/io/trino/plugin/hive/NodeVersion.java rename to core/trino-spi/src/main/java/io/trino/spi/NodeVersion.java index f8484ce72995..daf0b5f5acf9 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/NodeVersion.java +++ b/core/trino-spi/src/main/java/io/trino/spi/NodeVersion.java @@ -11,7 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.hive; +package io.trino.spi; import static java.util.Objects.requireNonNull; diff --git a/core/trino-spi/src/main/java/io/trino/spi/PageSorter.java b/core/trino-spi/src/main/java/io/trino/spi/PageSorter.java index 06f23c21ab62..f87951743e0c 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/PageSorter.java +++ b/core/trino-spi/src/main/java/io/trino/spi/PageSorter.java @@ -16,17 +16,13 @@ import io.trino.spi.connector.SortOrder; import io.trino.spi.type.Type; +import java.util.Iterator; import java.util.List; public interface PageSorter { /** - * @return Sorted synthetic addresses for pages. A synthetic address is encoded as a long with - * the high 32 bits containing the page index and the low 32 bits containing position index + * @return Iterator of sorted pages. */ - long[] sort(List types, List pages, List sortChannels, List sortOrders, int expectedPositions); - - int decodePageIndex(long address); - - int decodePositionIndex(long address); + Iterator sort(List types, List pages, List sortChannels, List sortOrders, int expectedPositions); } diff --git a/core/trino-spi/src/main/java/io/trino/spi/QueryId.java b/core/trino-spi/src/main/java/io/trino/spi/QueryId.java index 3ea4f1b3ba93..3136dbac76ae 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/QueryId.java +++ b/core/trino-spi/src/main/java/io/trino/spi/QueryId.java @@ -15,16 +15,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; -import com.google.errorprone.annotations.FormatMethod; import java.util.List; -import java.util.Objects; -import static java.lang.String.format; +import static io.airlift.slice.SizeOf.estimatedSizeOf; +import static io.airlift.slice.SizeOf.instanceSize; import static java.util.Objects.requireNonNull; -public final class QueryId +public record QueryId(String id) { + private static final int INSTANCE_SIZE = instanceSize(QueryId.class); + @JsonCreator public static QueryId valueOf(String queryId) { @@ -32,48 +33,30 @@ public static QueryId valueOf(String queryId) return new QueryId(queryId); } - private final String id; - - public QueryId(String id) + public QueryId { - this.id = validateId(id); + requireNonNull(id, "id is null"); + checkArgument(!id.isEmpty(), "id is empty"); + validateId(id); } + // For backward compatibility + @JsonValue + @Deprecated // Use id() instead public String getId() { return id; } @Override - @JsonValue public String toString() { return id; } - @Override - public int hashCode() - { - return Objects.hash(id); - } - - @Override - public boolean equals(Object obj) - { - if (this == obj) { - return true; - } - if (obj == null || getClass() != obj.getClass()) { - return false; - } - QueryId other = (QueryId) obj; - return Objects.equals(this.id, other.id); - } - // // Id helper methods // - // Check if the string matches [_a-z0-9]+ , but without the overhead of regex private static boolean isValidId(String id) { @@ -86,34 +69,62 @@ private static boolean isValidId(String id) return true; } + private static boolean isValidDottedId(char[] chars) + { + for (int i = 0; i < chars.length; i++) { + if (!(chars[i] == '_' || chars[i] == '.' || chars[i] >= 'a' && chars[i] <= 'z' || chars[i] >= '0' && chars[i] <= '9')) { + return false; + } + } + return true; + } + public static String validateId(String id) { - requireNonNull(id, "id is null"); - checkArgument(!id.isEmpty(), "id is empty"); - checkArgument(isValidId(id), "Invalid id %s", id); + if (!isValidId(id)) { + throw new IllegalArgumentException("Invalid queryId " + id); + } return id; } public static List parseDottedId(String id, int expectedParts, String name) { requireNonNull(id, "id is null"); - checkArgument(expectedParts > 0, "expectedParts must be at least 1"); + checkArgument(expectedParts > 1, "expectedParts must be at least 2"); requireNonNull(name, "name is null"); - List ids = List.of(id.split("\\.")); - checkArgument(ids.size() == expectedParts, "Invalid %s %s", name, id); - - for (String part : ids) { - validateId(part); + char[] chars = id.toCharArray(); + if (!isValidDottedId(chars)) { + throw new IllegalArgumentException("Invalid " + name + " " + id); + } + String[] parts = new String[expectedParts]; + int startOffset = 0; + int partIndex = 0; + for (int i = 0, length = chars.length; i < length; i++) { + if (chars[i] == '.') { + if (i <= startOffset || i == length - 1) { + throw new IllegalArgumentException("Invalid " + name + " " + id); + } + parts[partIndex++] = new String(chars, startOffset, i - startOffset); + startOffset = i + 1; + } } - return ids; + parts[partIndex++] = new String(chars, startOffset, chars.length - startOffset); + if (partIndex != expectedParts) { + throw new IllegalArgumentException("Invalid " + name + " " + id); + } + return List.of(parts); } - @FormatMethod - private static void checkArgument(boolean condition, String message, Object... messageArgs) + private static void checkArgument(boolean condition, String message) { if (!condition) { - throw new IllegalArgumentException(format(message, messageArgs)); + throw new IllegalArgumentException(message); } } + + public long getRetainedSizeInBytes() + { + return INSTANCE_SIZE + estimatedSizeOf(id); + } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/ArrayBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/ArrayBlockBuilder.java index 2719632a20e1..53879c21bf27 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/ArrayBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/ArrayBlockBuilder.java @@ -21,7 +21,6 @@ import static io.airlift.slice.SizeOf.instanceSize; import static io.airlift.slice.SizeOf.sizeOf; import static io.trino.spi.block.ArrayBlock.createArrayBlockInternal; -import static io.trino.spi.block.BlockUtil.appendRawBlockRange; import static io.trino.spi.block.BlockUtil.calculateNewArraySize; import static java.lang.Math.max; import static java.util.Objects.checkIndex; @@ -138,7 +137,7 @@ public void append(ValueBlock block, int position) int startOffset = offsets[offsetBase + position]; int length = offsets[offsetBase + position + 1] - startOffset; - appendRawBlockRange(arrayBlock.getRawElementBlock(), startOffset, length, values); + values.appendBlockRange(arrayBlock.getRawElementBlock(), startOffset, length); entryAdded(false); } @@ -169,7 +168,7 @@ public void appendRange(ValueBlock block, int offset, int length) int startOffset = rawOffsets[rawOffsetBase + offset]; int endOffset = rawOffsets[rawOffsetBase + offset + length]; - appendRawBlockRange(arrayBlock.getRawElementBlock(), startOffset, endOffset - startOffset, values); + values.appendBlockRange(arrayBlock.getRawElementBlock(), startOffset, endOffset - startOffset); // update offsets for copied data for (int i = 0; i < length; i++) { @@ -180,12 +179,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = arrayBlock.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawOffsetBase + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawOffsetBase + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawOffsetBase + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/BlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/BlockBuilder.java index 32d9950c53dd..02712b299270 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/BlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/BlockBuilder.java @@ -55,6 +55,19 @@ public interface BlockBuilder */ void appendPositions(ValueBlock block, int[] positions, int offset, int length); + /** + * Append the values in the specified range of a Block, this should be used only when + * the Block type can be potentially something other than ValueBlock. + */ + default void appendBlockRange(Block rawBlock, int offset, int length) + { + switch (rawBlock) { + case RunLengthEncodedBlock rleBlock -> appendRepeated(rleBlock.getValue(), 0, length); + case DictionaryBlock dictionaryBlock -> appendPositions(dictionaryBlock.getDictionary(), dictionaryBlock.getRawIds(), dictionaryBlock.getRawIdsOffset() + offset, length); + case ValueBlock valueBlock -> appendRange(valueBlock, offset, length); + } + } + /** * Appends a null value to the block. */ diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/BlockUtil.java b/core/trino-spi/src/main/java/io/trino/spi/block/BlockUtil.java index ff63a0aa32c9..b7d28e362037 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/BlockUtil.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/BlockUtil.java @@ -319,15 +319,6 @@ else if (buffer.length < capacity) { return buffer; } - static void appendRawBlockRange(Block rawBlock, int offset, int length, BlockBuilder blockBuilder) - { - switch (rawBlock) { - case RunLengthEncodedBlock rleBlock -> blockBuilder.appendRepeated(rleBlock.getValue(), 0, length); - case DictionaryBlock dictionaryBlock -> blockBuilder.appendPositions(dictionaryBlock.getDictionary(), dictionaryBlock.getRawIds(), offset, length); - case ValueBlock valueBlock -> blockBuilder.appendRange(valueBlock, offset, length); - } - } - /** * Ideally, the underlying nulls array in Block implementations should be a byte array instead of a boolean array. * This method is used to perform that conversion until the Block implementations are changed. diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/ByteArrayBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/ByteArrayBlockBuilder.java index 9e459847dba6..cdbe20ce7f3d 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/ByteArrayBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/ByteArrayBlockBuilder.java @@ -140,12 +140,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = byteArrayBlock.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawOffset + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawOffset + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawOffset + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/EncoderUtil.java b/core/trino-spi/src/main/java/io/trino/spi/block/EncoderUtil.java index 4c9584da7f7c..2d6c181cdfe0 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/EncoderUtil.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/EncoderUtil.java @@ -68,52 +68,6 @@ public static void encodeNullsAsBits(SliceOutput sliceOutput, @Nullable boolean[ sliceOutput.writeBytes(packedIsNull, 0, currentByte); } - /** - * Append null values for the block as a stream of bits. - * - * @deprecated Use {@link EncoderUtil#encodeNullsAsBits(SliceOutput, boolean[], int, int)} instead - */ - @Deprecated(forRemoval = true) - @SuppressWarnings({"NarrowingCompoundAssignment", "ImplicitNumericConversion"}) - public static void encodeNullsAsBits(SliceOutput sliceOutput, Block block) - { - boolean mayHaveNull = block.mayHaveNull(); - sliceOutput.writeBoolean(mayHaveNull); - if (!mayHaveNull) { - return; - } - - int positionCount = block.getPositionCount(); - byte[] packedIsNull = new byte[((positionCount & ~0b111) + 1) / 8]; - int currentByte = 0; - - for (int position = 0; position < (positionCount & ~0b111); position += 8, currentByte++) { - byte value = 0; - value |= block.isNull(position) ? 0b1000_0000 : 0; - value |= block.isNull(position + 1) ? 0b0100_0000 : 0; - value |= block.isNull(position + 2) ? 0b0010_0000 : 0; - value |= block.isNull(position + 3) ? 0b0001_0000 : 0; - value |= block.isNull(position + 4) ? 0b0000_1000 : 0; - value |= block.isNull(position + 5) ? 0b0000_0100 : 0; - value |= block.isNull(position + 6) ? 0b0000_0010 : 0; - value |= block.isNull(position + 7) ? 0b0000_0001 : 0; - packedIsNull[currentByte] = value; - } - - sliceOutput.writeBytes(packedIsNull); - - // write last null bits - if ((positionCount & 0b111) > 0) { - byte value = 0; - int mask = 0b1000_0000; - for (int position = positionCount & ~0b111; position < positionCount; position++) { - value |= block.isNull(position) ? mask : 0; - mask >>>= 1; - } - sliceOutput.appendByte(value); - } - } - /** * Decode the bit stream created by encodeNullsAsBits. */ diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/Fixed12BlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/Fixed12BlockBuilder.java index 10506973a43e..e600f3d96e04 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/Fixed12BlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/Fixed12BlockBuilder.java @@ -158,12 +158,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = fixed12Block.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawOffset + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawOffset + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawOffset + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/Int128ArrayBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/Int128ArrayBlockBuilder.java index 5c57e5d5032f..197d3eca4102 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/Int128ArrayBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/Int128ArrayBlockBuilder.java @@ -154,12 +154,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = int128ArrayBlock.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawOffset + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawOffset + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawOffset + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/IntArrayBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/IntArrayBlockBuilder.java index 8508c0ec79f1..6e3284580ae3 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/IntArrayBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/IntArrayBlockBuilder.java @@ -139,12 +139,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = intArrayBlock.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawOffset + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawOffset + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawOffset + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/LongArrayBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/LongArrayBlockBuilder.java index cff5d98a0dce..937de1553069 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/LongArrayBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/LongArrayBlockBuilder.java @@ -139,12 +139,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = longArrayBlock.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawOffset + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawOffset + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawOffset + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/MapBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/MapBlockBuilder.java index cf36bb4a381a..5dd1a66a765e 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/MapBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/MapBlockBuilder.java @@ -23,7 +23,6 @@ import static io.airlift.slice.SizeOf.instanceSize; import static io.airlift.slice.SizeOf.sizeOf; -import static io.trino.spi.block.BlockUtil.appendRawBlockRange; import static io.trino.spi.block.BlockUtil.calculateNewArraySize; import static io.trino.spi.block.MapBlock.createMapBlockInternal; import static io.trino.spi.block.MapHashTables.HASH_MULTIPLIER; @@ -152,8 +151,8 @@ public void append(ValueBlock block, int position) int startOffset = offsets[offsetBase + position]; int length = offsets[offsetBase + position + 1] - startOffset; - appendRawBlockRange(mapBlock.getRawKeyBlock(), startOffset, length, keyBlockBuilder); - appendRawBlockRange(mapBlock.getRawValueBlock(), startOffset, length, valueBlockBuilder); + keyBlockBuilder.appendBlockRange(mapBlock.getRawKeyBlock(), startOffset, length); + valueBlockBuilder.appendBlockRange(mapBlock.getRawValueBlock(), startOffset, length); entryAdded(false); } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/RowBlock.java b/core/trino-spi/src/main/java/io/trino/spi/block/RowBlock.java index e8ff5c975dde..39a9f9798186 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/RowBlock.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/RowBlock.java @@ -185,7 +185,7 @@ public boolean hasNull() } @Nullable - boolean[] getRawRowIsNull() + public boolean[] getRawRowIsNull() { return rowIsNull; } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/RowBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/RowBlockBuilder.java index b7c4f8806843..003e95f069d6 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/RowBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/RowBlockBuilder.java @@ -174,12 +174,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawRowIsNull = rowBlock.getRawRowIsNull(); if (rawRowIsNull != null) { for (int i = 0; i < length; i++) { - if (rawRowIsNull[startOffset + offset + i]) { - rowIsNull[positionCount + i] = true; - hasNullRow = true; + boolean isNull = rawRowIsNull[startOffset + offset + i]; + hasNullRow |= isNull; + hasNonNullRow |= !isNull; + if (hasNullRow & hasNonNullRow) { + System.arraycopy(rawRowIsNull, startOffset + offset + i, rowIsNull, positionCount + i, length - i); + break; } else { - hasNonNullRow = true; + rowIsNull[positionCount + i] = isNull; } } } @@ -276,7 +279,7 @@ public void appendPositions(ValueBlock block, int[] positions, int offset, int l if (startOffset == 0) { for (int fieldId = 0; fieldId < fieldBlockBuilders.length; fieldId++) { - appendPositionsToField(rawFieldBlocks[fieldId], positions, 0, length, fieldBlockBuilders[fieldId]); + appendPositionsToField(rawFieldBlocks[fieldId], positions, offset, length, fieldBlockBuilders[fieldId]); } } else { diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/ShortArrayBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/ShortArrayBlockBuilder.java index 263ac46716df..08f36d8f9125 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/ShortArrayBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/ShortArrayBlockBuilder.java @@ -139,12 +139,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = shortArrayBlock.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawOffset + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawOffset + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawOffset + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/block/VariableWidthBlockBuilder.java b/core/trino-spi/src/main/java/io/trino/spi/block/VariableWidthBlockBuilder.java index 571e76b9fa01..89a8a599a90c 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/block/VariableWidthBlockBuilder.java +++ b/core/trino-spi/src/main/java/io/trino/spi/block/VariableWidthBlockBuilder.java @@ -244,12 +244,15 @@ public void appendRange(ValueBlock block, int offset, int length) boolean[] rawValueIsNull = variableWidthBlock.getRawValueIsNull(); if (rawValueIsNull != null) { for (int i = 0; i < length; i++) { - if (rawValueIsNull[rawArrayBase + offset + i]) { - valueIsNull[positionCount + i] = true; - hasNullValue = true; + boolean isNull = rawValueIsNull[rawArrayBase + offset + i]; + hasNullValue |= isNull; + hasNonNullValue |= !isNull; + if (hasNullValue & hasNonNullValue) { + System.arraycopy(rawValueIsNull, rawArrayBase + offset + i, valueIsNull, positionCount + i, length - i); + break; } else { - hasNonNullValue = true; + valueIsNull[positionCount + i] = isNull; } } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/CatalogVersion.java b/core/trino-spi/src/main/java/io/trino/spi/connector/CatalogVersion.java index 934b53b7b9df..475f8f417fec 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/CatalogVersion.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/CatalogVersion.java @@ -44,9 +44,9 @@ public record CatalogVersion(String version) private static boolean isAllowedCharacter(char c) { return ('0' <= c && c <= '9') || - ('a' <= c && c <= 'z') || - c == '_' || - c == '-'; + ('a' <= c && c <= 'z') || + c == '_' || + c == '-'; } @JsonValue diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java index 8f32252f898c..055a1605be09 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorMetadata.java @@ -28,6 +28,7 @@ import io.trino.spi.function.LanguageFunction; import io.trino.spi.function.SchemaFunctionName; import io.trino.spi.function.table.ConnectorTableFunctionHandle; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.GrantInfo; import io.trino.spi.security.Privilege; @@ -112,7 +113,11 @@ default ConnectorTableHandle getTableHandle( Optional startVersion, Optional endVersion) { - throw new TrinoException(GENERIC_INTERNAL_ERROR, "ConnectorMetadata getTableHandle() is not implemented"); + if (listTables(session, Optional.of(tableName.getSchemaName())).isEmpty()) { + // This is a correct default implementation meant primarily for connectors that do not have any tables. + return null; + } + throw new TrinoException(GENERIC_INTERNAL_ERROR, "ConnectorMetadata listTables() is implemented without getTableHandle()"); } /** @@ -160,9 +165,10 @@ default void finishTableExecute(ConnectorSession session, ConnectorTableExecuteH } /** - * Execute a {@link TableProcedureExecutionMode#coordinatorOnly() coordinator-only} table procedure. + * Execute a {@link TableProcedureExecutionMode#coordinatorOnly() coordinator-only} table procedure + * and return procedure execution metrics that will be populated in the query output. */ - default void executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) + default Map executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) { throw new TrinoException(GENERIC_INTERNAL_ERROR, "ConnectorMetadata executeTableExecute() is not implemented"); } @@ -248,6 +254,14 @@ default Optional getInfo(ConnectorSession session, ConnectorTableHandle return Optional.empty(); } + /** + * Return connector-specific, metadata operations metrics for the given session. + */ + default Metrics getMetrics(ConnectorSession session) + { + return Metrics.EMPTY; + } + /** * List table, view and materialized view names, possibly filtered by schema. An empty list is returned if none match. * An empty list is returned also when schema name does not refer to an existing schema. @@ -683,7 +697,20 @@ default Optional getInsertLayout(ConnectorSession session, /** * Describes statistics that must be collected during a write. + * + * @param tableReplace indicates whether this is for table replace operation and statistics of an existing table (if any) should be ignored */ + default TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean tableReplace) + { + return getStatisticsCollectionMetadataForWrite(session, tableMetadata); + } + + /** + * Describes statistics that must be collected during a write. + * + * @deprecated use {@link #getStatisticsCollectionMetadataForWrite(ConnectorSession, ConnectorTableMetadata, boolean)} + */ + @Deprecated default TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) { return TableStatisticsMetadata.empty(); @@ -723,7 +750,6 @@ default void finishStatisticsCollection(ConnectorSession session, ConnectorTable */ default ConnectorOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, Optional layout, RetryMode retryMode, boolean replace) { - // Redirect to deprecated SPI to not break existing connectors if (replace) { throw new TrinoException(NOT_SUPPORTED, "This connector does not support replacing tables"); } diff --git a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorSplit.java b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorSplit.java index d306d11951d5..8119183c723e 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorSplit.java +++ b/core/trino-spi/src/main/java/io/trino/spi/connector/ConnectorSplit.java @@ -13,12 +13,10 @@ */ package io.trino.spi.connector; -import com.fasterxml.jackson.annotation.JsonIgnore; import io.trino.spi.HostAddress; import io.trino.spi.SplitWeight; import java.util.List; -import java.util.Map; public interface ConnectorSplit { @@ -42,16 +40,6 @@ default List getAddresses() return List.of(); } - /** - * @deprecated Use {@link Object#toString()} for printing debugging information, or {@link ConnectorSplitSource#getMetrics()} for recording metrics - */ - @Deprecated(forRemoval = true) - @JsonIgnore // ConnectorSplit is json-serializable, but we don't want to repeat information in that field - default Map getSplitInfo() - { - throw new UnsupportedOperationException(); - } - default SplitWeight getSplitWeight() { return SplitWeight.standard(); diff --git a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java index 7f600c11aa2f..8f501ffd4eb6 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java +++ b/core/trino-spi/src/main/java/io/trino/spi/eventlistener/QueryStatistics.java @@ -16,9 +16,11 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.trino.spi.Unstable; +import io.trino.spi.metrics.Metrics; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Supplier; @@ -46,6 +48,7 @@ public class QueryStatistics private final Optional outputBlockedTime; private final Optional failedOutputBlockedTime; private final Optional physicalInputReadTime; + private final Optional finishingTime; private final long peakUserMemoryBytes; private final long peakTaskUserMemory; @@ -82,6 +85,7 @@ public class QueryStatistics */ private final Supplier> operatorSummariesProvider; private final List optimizerRulesSummaries; + private final Map catalogMetadataMetrics; /** * Plan node stats and costs serialized to JSON. Serialization format and structure * can change without preserving backward compatibility. @@ -108,6 +112,7 @@ public QueryStatistics( Optional outputBlockedTime, Optional failedOutputBlockedTime, Optional physicalInputReadTime, + Optional finishingTime, long peakUserMemoryBytes, long peakTaskUserMemory, long peakTaskTotalMemory, @@ -134,6 +139,7 @@ public QueryStatistics( List dynamicFilterDomainStatistics, List operatorSummaries, List optimizerRulesSummaries, + Map catalogMetadataMetrics, Optional planNodeStatsAndCosts) { this( @@ -154,6 +160,7 @@ public QueryStatistics( outputBlockedTime, failedOutputBlockedTime, physicalInputReadTime, + finishingTime, peakUserMemoryBytes, peakTaskUserMemory, peakTaskTotalMemory, @@ -180,6 +187,7 @@ public QueryStatistics( dynamicFilterDomainStatistics, () -> operatorSummaries, optimizerRulesSummaries, + catalogMetadataMetrics, planNodeStatsAndCosts); } @@ -201,6 +209,7 @@ public QueryStatistics( Optional outputBlockedTime, Optional failedOutputBlockedTime, Optional physicalInputReadTime, + Optional finishingTime, long peakUserMemoryBytes, long peakTaskUserMemory, long peakTaskTotalMemory, @@ -227,6 +236,7 @@ public QueryStatistics( List dynamicFilterDomainStatistics, Supplier> operatorSummariesProvider, List optimizerRulesSummaries, + Map catalogMetadataMetrics, Optional planNodeStatsAndCosts) { this.cpuTime = requireNonNull(cpuTime, "cpuTime is null"); @@ -246,6 +256,7 @@ public QueryStatistics( this.outputBlockedTime = requireNonNull(outputBlockedTime, "outputBlockedTime is null"); this.failedOutputBlockedTime = requireNonNull(failedOutputBlockedTime, "failedOutputBlockedTime is null"); this.physicalInputReadTime = requireNonNull(physicalInputReadTime, "physicalInputReadTime is null"); + this.finishingTime = requireNonNull(finishingTime, "finishingTime is null"); this.peakUserMemoryBytes = peakUserMemoryBytes; this.peakTaskUserMemory = peakTaskUserMemory; this.peakTaskTotalMemory = peakTaskTotalMemory; @@ -272,6 +283,7 @@ public QueryStatistics( this.dynamicFilterDomainStatistics = requireNonNull(dynamicFilterDomainStatistics, "dynamicFilterDomainStatistics is null"); this.operatorSummariesProvider = requireNonNull(operatorSummariesProvider, "operatorSummariesProvider is null"); this.optimizerRulesSummaries = requireNonNull(optimizerRulesSummaries, "optimizerRulesSummaries is null"); + this.catalogMetadataMetrics = requireNonNull(catalogMetadataMetrics, "catalogMetadataMetrics is null"); this.planNodeStatsAndCosts = requireNonNull(planNodeStatsAndCosts, "planNodeStatsAndCosts is null"); } @@ -377,6 +389,12 @@ public Optional getPhysicalInputReadTime() return physicalInputReadTime; } + @JsonProperty + public Optional getFinishingTime() + { + return finishingTime; + } + @JsonProperty public long getPeakUserMemoryBytes() { @@ -533,6 +551,12 @@ public List getOptimizerRulesSummaries() return optimizerRulesSummaries; } + @JsonProperty + public Map getCatalogMetadataMetrics() + { + return catalogMetadataMetrics; + } + @JsonProperty public Optional getPlanNodeStatsAndCosts() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/metrics/Distribution.java b/core/trino-spi/src/main/java/io/trino/spi/metrics/Distribution.java index 3c7a6ae3f670..f675636539f0 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/metrics/Distribution.java +++ b/core/trino-spi/src/main/java/io/trino/spi/metrics/Distribution.java @@ -22,5 +22,5 @@ public interface Distribution double getMax(); - double getPercentile(double percentile); + double[] getPercentiles(double... percentiles); } diff --git a/core/trino-spi/src/main/java/io/trino/spi/predicate/SortedRangeSet.java b/core/trino-spi/src/main/java/io/trino/spi/predicate/SortedRangeSet.java index 05fb6a38ff2b..c1bec7a223f6 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/predicate/SortedRangeSet.java +++ b/core/trino-spi/src/main/java/io/trino/spi/predicate/SortedRangeSet.java @@ -641,7 +641,7 @@ SortedRangeSet linearSearchIntersect(SortedRangeSet that) Optional intersect = thisCurrent.tryIntersect(thatCurrent); if (intersect.isPresent()) { - writeRange(type, blockBuilder, inclusive, resultRangeIndex, intersect.get()); + writeRange(blockBuilder, inclusive, resultRangeIndex, intersect.get()); resultRangeIndex++; } int compare = thisCurrent.compareHighBound(thatCurrent); @@ -728,7 +728,7 @@ SortedRangeSet binarySearchIntersect(SortedRangeSet that) if (probeIndex == insertionStartIndex || probeIndex + 1 >= intersectionEndIndex) { Optional intersect = probeRange.tryIntersect(current); if (intersect.isPresent()) { - writeRange(type, blockBuilder, inclusive, resultIndex, intersect.get()); + writeRange(blockBuilder, inclusive, resultIndex, intersect.get()); resultIndex++; } probeIndex++; @@ -1026,7 +1026,7 @@ else if (thatNextRangeIndex == thatRangeCount) { current = merged.get(); } else { - writeRange(type, blockBuilder, inclusive, resultRangeIndex, current); + writeRange(blockBuilder, inclusive, resultRangeIndex, current); resultRangeIndex++; current = next; } @@ -1036,7 +1036,7 @@ else if (thatNextRangeIndex == thatRangeCount) { } } if (current != null) { - writeRange(type, blockBuilder, inclusive, resultRangeIndex, current); + writeRange(blockBuilder, inclusive, resultRangeIndex, current); resultRangeIndex++; } @@ -1207,7 +1207,7 @@ public SortedRangeSet complement() inclusive[2 * resultRangeIndex] = false; inclusive[2 * resultRangeIndex + 1] = !first.lowInclusive; blockBuilder.appendNull(); - type.appendTo(first.lowValueBlock, first.lowValuePosition, blockBuilder); + blockBuilder.append(first.lowValueBlock.getUnderlyingValueBlock(), first.lowValueBlock.getUnderlyingValuePosition(first.lowValuePosition)); resultRangeIndex++; } @@ -1217,8 +1217,8 @@ public SortedRangeSet complement() inclusive[2 * resultRangeIndex] = !previous.highInclusive; inclusive[2 * resultRangeIndex + 1] = !current.lowInclusive; - type.appendTo(previous.highValueBlock, previous.highValuePosition, blockBuilder); - type.appendTo(current.lowValueBlock, current.lowValuePosition, blockBuilder); + blockBuilder.append(previous.highValueBlock.getUnderlyingValueBlock(), previous.highValueBlock.getUnderlyingValuePosition(previous.highValuePosition)); + blockBuilder.append(current.lowValueBlock.getUnderlyingValueBlock(), current.lowValueBlock.getUnderlyingValuePosition(current.lowValuePosition)); resultRangeIndex++; previous = current; @@ -1227,7 +1227,7 @@ public SortedRangeSet complement() if (!last.isHighUnbounded()) { inclusive[2 * resultRangeIndex] = !last.highInclusive; inclusive[2 * resultRangeIndex + 1] = false; - type.appendTo(last.highValueBlock, last.highValuePosition, blockBuilder); + blockBuilder.append(last.highValueBlock.getUnderlyingValueBlock(), last.highValueBlock.getUnderlyingValuePosition(last.highValuePosition)); blockBuilder.appendNull(); resultRangeIndex++; } @@ -1529,12 +1529,12 @@ private static void writeRange(Type type, BlockBuilder blockBuilder, boolean[] i writeNativeValue(type, blockBuilder, range.getHighValue().orElse(null)); } - private static void writeRange(Type type, BlockBuilder blockBuilder, boolean[] inclusive, int rangeIndex, RangeView range) + private static void writeRange(BlockBuilder blockBuilder, boolean[] inclusive, int rangeIndex, RangeView range) { inclusive[2 * rangeIndex] = range.lowInclusive; inclusive[2 * rangeIndex + 1] = range.highInclusive; - type.appendTo(range.lowValueBlock, range.lowValuePosition, blockBuilder); - type.appendTo(range.highValueBlock, range.highValuePosition, blockBuilder); + blockBuilder.append(range.lowValueBlock.getUnderlyingValueBlock(), range.lowValueBlock.getUnderlyingValuePosition(range.lowValuePosition)); + blockBuilder.append(range.highValueBlock.getUnderlyingValueBlock(), range.highValueBlock.getUnderlyingValuePosition(range.highValuePosition)); } private static void checkNotNaN(Type type, Object value) diff --git a/core/trino-spi/src/main/java/io/trino/spi/resourcegroups/SelectionCriteria.java b/core/trino-spi/src/main/java/io/trino/spi/resourcegroups/SelectionCriteria.java index 11397f5606a7..20cd7b5a2d10 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/resourcegroups/SelectionCriteria.java +++ b/core/trino-spi/src/main/java/io/trino/spi/resourcegroups/SelectionCriteria.java @@ -55,31 +55,6 @@ public SelectionCriteria( this.queryType = requireNonNull(queryType, "queryType is null"); } - /** - * @deprecated Use {@link #SelectionCriteria(boolean, String, Set, String, Optional, Optional, Set, ResourceEstimates, Optional)} instead. - */ - @Deprecated(since = "474", forRemoval = true) - public SelectionCriteria( - boolean authenticated, - String user, - Set userGroups, - Optional source, - Set clientTags, - ResourceEstimates resourceEstimates, - Optional queryType) - { - this( - authenticated, - user, - userGroups, - user, - Optional.empty(), - source, - clientTags, - resourceEstimates, - queryType); - } - public boolean isAuthenticated() { return authenticated; diff --git a/core/trino-spi/src/main/java/io/trino/spi/security/PrivilegeInfo.java b/core/trino-spi/src/main/java/io/trino/spi/security/PrivilegeInfo.java index 87e71ec4ea9c..911c9608554d 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/security/PrivilegeInfo.java +++ b/core/trino-spi/src/main/java/io/trino/spi/security/PrivilegeInfo.java @@ -53,6 +53,6 @@ public boolean equals(Object o) } PrivilegeInfo privilegeInfo = (PrivilegeInfo) o; return privilege == privilegeInfo.privilege && - grantOption == privilegeInfo.grantOption; + grantOption == privilegeInfo.grantOption; } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/AbstractIntType.java b/core/trino-spi/src/main/java/io/trino/spi/type/AbstractIntType.java index a544ee8efb28..b17f6e3fd96e 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/AbstractIntType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/AbstractIntType.java @@ -114,17 +114,6 @@ protected void checkValueValid(long value) } } - @Override - public final void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeInt(blockBuilder, getInt(block, position)); - } - } - @Override public int getFlatFixedSize() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/AbstractLongType.java b/core/trino-spi/src/main/java/io/trino/spi/type/AbstractLongType.java index e0cefd3f8a2c..fc82b2081ee1 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/AbstractLongType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/AbstractLongType.java @@ -91,17 +91,6 @@ public final void writeLong(BlockBuilder blockBuilder, long value) ((LongArrayBlockBuilder) blockBuilder).writeLong(value); } - @Override - public final void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeLong(blockBuilder, getLong(block, position)); - } - } - @Override public int getFlatFixedSize() { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/AbstractVariableWidthType.java b/core/trino-spi/src/main/java/io/trino/spi/type/AbstractVariableWidthType.java index 6639b2fa1cc7..095fbec38bdc 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/AbstractVariableWidthType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/AbstractVariableWidthType.java @@ -88,22 +88,6 @@ public VariableWidthBlockBuilder createBlockBuilder(BlockBuilderStatus blockBuil return createBlockBuilder(blockBuilderStatus, expectedEntries, EXPECTED_BYTES_PER_ENTRY); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - VariableWidthBlock variableWidthBlock = (VariableWidthBlock) block.getUnderlyingValueBlock(); - position = block.getUnderlyingValuePosition(position); - Slice slice = variableWidthBlock.getRawSlice(); - int offset = variableWidthBlock.getRawSliceOffset(position); - int length = variableWidthBlock.getSliceLength(position); - ((VariableWidthBlockBuilder) blockBuilder).writeEntry(slice, offset, length); - } - } - @Override public TypeOperatorDeclaration getTypeOperatorDeclaration(TypeOperators typeOperators) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/ArrayType.java b/core/trino-spi/src/main/java/io/trino/spi/type/ArrayType.java index 098ba44fedc6..d19543c33ccd 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/ArrayType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/ArrayType.java @@ -244,17 +244,6 @@ private List arrayBlockToObjectValues(Block block, int start, int length return Collections.unmodifiableList(values); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeObject(blockBuilder, getObject(block, position)); - } - } - @Override public Block getObject(Block block, int position) { @@ -265,11 +254,8 @@ public Block getObject(Block block, int position) public void writeObject(BlockBuilder blockBuilder, Object value) { Block arrayBlock = (Block) value; - ((ArrayBlockBuilder) blockBuilder).buildEntry(elementBuilder -> { - for (int i = 0; i < arrayBlock.getPositionCount(); i++) { - elementType.appendTo(arrayBlock, i, elementBuilder); - } - }); + ((ArrayBlockBuilder) blockBuilder).buildEntry(elementBuilder -> + elementBuilder.appendBlockRange(arrayBlock, 0, arrayBlock.getPositionCount())); } // FLAT MEMORY LAYOUT diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/BooleanType.java b/core/trino-spi/src/main/java/io/trino/spi/type/BooleanType.java index 812da90f8f9f..0fb118304ce6 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/BooleanType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/BooleanType.java @@ -127,17 +127,6 @@ public Object getObjectValue(Block block, int position) return getBoolean(block, position); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - ((ByteArrayBlockBuilder) blockBuilder).writeByte(getBoolean(block, position) ? (byte) 1 : 0); - } - } - @Override public boolean getBoolean(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/DoubleType.java b/core/trino-spi/src/main/java/io/trino/spi/type/DoubleType.java index 9d8d44e493c7..af4dbc34f425 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/DoubleType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/DoubleType.java @@ -95,19 +95,6 @@ public Object getObjectValue(Block block, int position) return getDouble(block, position); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - LongArrayBlock valueBlock = (LongArrayBlock) block.getUnderlyingValueBlock(); - int valuePosition = block.getUnderlyingValuePosition(position); - ((LongArrayBlockBuilder) blockBuilder).writeLong(valueBlock.getLong(valuePosition)); - } - } - @Override public double getDouble(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/LongDecimalType.java b/core/trino-spi/src/main/java/io/trino/spi/type/LongDecimalType.java index 515832b1fc0d..a5c91a7cdb06 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/LongDecimalType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/LongDecimalType.java @@ -99,19 +99,6 @@ public Object getObjectValue(Block block, int position) return new SqlDecimal(unscaledValue, getPrecision(), getScale()); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - Int128ArrayBlock valueBlock = (Int128ArrayBlock) block.getUnderlyingValueBlock(); - int valuePosition = block.getUnderlyingValuePosition(position); - ((Int128ArrayBlockBuilder) blockBuilder).writeInt128(valueBlock.getInt128High(valuePosition), valueBlock.getInt128Low(valuePosition)); - } - } - @Override public void writeObject(BlockBuilder blockBuilder, Object value) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/LongTimeWithTimeZoneType.java b/core/trino-spi/src/main/java/io/trino/spi/type/LongTimeWithTimeZoneType.java index 91c1de0fe701..95ff1097d40f 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/LongTimeWithTimeZoneType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/LongTimeWithTimeZoneType.java @@ -94,19 +94,6 @@ public BlockBuilder createFixedSizeBlockBuilder(int positionCount) return new Fixed12BlockBuilder(null, positionCount); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - Fixed12Block valueBlock = (Fixed12Block) block.getUnderlyingValueBlock(); - int valuePosition = block.getUnderlyingValuePosition(position); - write(blockBuilder, getPicos(valueBlock, valuePosition), getOffsetMinutes(valueBlock, valuePosition)); - } - } - @Override public LongTimeWithTimeZone getObject(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampType.java b/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampType.java index 6539ce42ed4d..5f23f18ae49b 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampType.java @@ -106,19 +106,6 @@ public BlockBuilder createFixedSizeBlockBuilder(int positionCount) return new Fixed12BlockBuilder(null, positionCount); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - Fixed12Block valueBlock = (Fixed12Block) block.getUnderlyingValueBlock(); - int valuePosition = block.getUnderlyingValuePosition(position); - ((Fixed12BlockBuilder) blockBuilder).writeFixed12(getEpochMicros(valueBlock, valuePosition), getFraction(valueBlock, valuePosition)); - } - } - @Override public Object getObject(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampWithTimeZoneType.java b/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampWithTimeZoneType.java index 68134c0a799f..0690f343bbc3 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampWithTimeZoneType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/LongTimestampWithTimeZoneType.java @@ -105,19 +105,6 @@ public BlockBuilder createFixedSizeBlockBuilder(int positionCount) return new Fixed12BlockBuilder(null, positionCount); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - Fixed12Block valueBlock = (Fixed12Block) block.getUnderlyingValueBlock(); - int valuePosition = block.getUnderlyingValuePosition(position); - write(blockBuilder, getPackedEpochMillis(valueBlock, valuePosition), getPicosOfMilli(valueBlock, valuePosition)); - } - } - @Override public Object getObject(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/MapType.java b/core/trino-spi/src/main/java/io/trino/spi/type/MapType.java index 11d059762142..b5d2928822b3 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/MapType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/MapType.java @@ -303,17 +303,6 @@ public Object getObjectValue(Block block, int position) return Collections.unmodifiableMap(map); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeObject(blockBuilder, getObject(block, position)); - } - } - @Override public SqlMap getObject(Block block, int position) { @@ -332,10 +321,8 @@ public void writeObject(BlockBuilder blockBuilder, Object value) Block rawValueBlock = sqlMap.getRawValueBlock(); ((MapBlockBuilder) blockBuilder).buildEntry((keyBuilder, valueBuilder) -> { - for (int i = 0; i < sqlMap.getSize(); i++) { - keyType.appendTo(rawKeyBlock, rawOffset + i, keyBuilder); - valueType.appendTo(rawValueBlock, rawOffset + i, valueBuilder); - } + keyBuilder.appendBlockRange(rawKeyBlock, rawOffset, sqlMap.getSize()); + valueBuilder.appendBlockRange(rawValueBlock, rawOffset, sqlMap.getSize()); }); } diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/RowType.java b/core/trino-spi/src/main/java/io/trino/spi/type/RowType.java index b0daa06d5de7..4e1d89a94389 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/RowType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/RowType.java @@ -249,17 +249,6 @@ public Object getObjectValue(Block block, int position) return Collections.unmodifiableList(values); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeObject(blockBuilder, getObject(block, position)); - } - } - @Override public SqlRow getObject(Block block, int position) { @@ -273,7 +262,8 @@ public void writeObject(BlockBuilder blockBuilder, Object value) int rawIndex = sqlRow.getRawIndex(); ((RowBlockBuilder) blockBuilder).buildEntry(fieldBuilders -> { for (int i = 0; i < sqlRow.getFieldCount(); i++) { - fields.get(i).getType().appendTo(sqlRow.getRawFieldBlock(i), rawIndex, fieldBuilders.get(i)); + Block block = sqlRow.getRawFieldBlock(i); + fieldBuilders.get(i).append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(rawIndex)); } }); } diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/ShortDecimalType.java b/core/trino-spi/src/main/java/io/trino/spi/type/ShortDecimalType.java index 1f89b4731fe6..577bdf371813 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/ShortDecimalType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/ShortDecimalType.java @@ -126,17 +126,6 @@ public Object getObjectValue(Block block, int position) return new SqlDecimal(BigInteger.valueOf(getLong(block, position)), getPrecision(), getScale()); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeLong(blockBuilder, getLong(block, position)); - } - } - @Override public long getLong(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimeWithTimeZoneType.java b/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimeWithTimeZoneType.java index 28ae4726b8f4..d8adb098af4d 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimeWithTimeZoneType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimeWithTimeZoneType.java @@ -89,17 +89,6 @@ public void writeLong(BlockBuilder blockBuilder, long value) ((LongArrayBlockBuilder) blockBuilder).writeLong(value); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeLong(blockBuilder, getLong(block, position)); - } - } - @Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampType.java b/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampType.java index d86439e04dbc..c3d270c040c3 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampType.java @@ -100,17 +100,6 @@ public void writeLong(BlockBuilder blockBuilder, long value) ((LongArrayBlockBuilder) blockBuilder).writeLong(value); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeLong(blockBuilder, getLong(block, position)); - } - } - @Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampWithTimeZoneType.java b/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampWithTimeZoneType.java index 1e745aad3231..533e15f8016a 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampWithTimeZoneType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/ShortTimestampWithTimeZoneType.java @@ -89,17 +89,6 @@ public void writeLong(BlockBuilder blockBuilder, long value) ((LongArrayBlockBuilder) blockBuilder).writeLong(value); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeLong(blockBuilder, getLong(block, position)); - } - } - @Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/SmallintType.java b/core/trino-spi/src/main/java/io/trino/spi/type/SmallintType.java index f4304959bc17..0f7b54898bec 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/SmallintType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/SmallintType.java @@ -151,17 +151,6 @@ public Optional> getDiscreteValues(Range range) return Optional.of(LongStream.rangeClosed((long) range.getMin(), (long) range.getMax()).boxed()); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - ((ShortArrayBlockBuilder) blockBuilder).writeShort(getShort(block, position)); - } - } - @Override public long getLong(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/TinyintType.java b/core/trino-spi/src/main/java/io/trino/spi/type/TinyintType.java index a85d014b1296..427c988912c2 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/TinyintType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/TinyintType.java @@ -147,17 +147,6 @@ public Optional> getDiscreteValues(Range range) return Optional.of(LongStream.rangeClosed((long) range.getMin(), (long) range.getMax()).boxed()); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - writeByte(blockBuilder, getByte(block, position)); - } - } - @Override public long getLong(Block block, int position) { diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/Type.java b/core/trino-spi/src/main/java/io/trino/spi/type/Type.java index d7f21f048b56..951ecf6edcbf 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/Type.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/Type.java @@ -183,9 +183,13 @@ default Object getObject(ConnectorSession ignored, Block block, int position) void writeObject(BlockBuilder blockBuilder, Object value); /** - * Append the value at {@code position} in {@code block} to {@code blockBuilder}. + * @deprecated Use {@link BlockBuilder#append} */ - void appendTo(Block block, int position, BlockBuilder blockBuilder); + @Deprecated(forRemoval = true) + default void appendTo(Block block, int position, BlockBuilder blockBuilder) + { + blockBuilder.append(block.getUnderlyingValueBlock(), block.getUnderlyingValuePosition(position)); + } /** * Return the range of possible values for this type, if available. diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/TypeOperators.java b/core/trino-spi/src/main/java/io/trino/spi/type/TypeOperators.java index 53389d456d59..0a3895f59663 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/TypeOperators.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/TypeOperators.java @@ -43,6 +43,7 @@ import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.FLAT; import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.NEVER_NULL; import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.NULL_FLAG; +import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.VALUE_BLOCK_POSITION_NOT_NULL; import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.BLOCK_BUILDER; import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL; import static io.trino.spi.function.InvocationConvention.simpleConvention; @@ -496,7 +497,12 @@ private OperatorMethodHandle generateOrderingOperator(OperatorConvention operato SortOrder sortOrder = operatorConvention.sortOrder().orElseThrow(() -> new IllegalArgumentException("Operator convention does not contain a sort order")); OperatorType comparisonType = operatorConvention.operatorType(); if (operatorConvention.callingConvention().getArgumentConventions().equals(List.of(BLOCK_POSITION, BLOCK_POSITION))) { - OperatorConvention comparisonOperator = new OperatorConvention(operatorConvention.type(), comparisonType, Optional.empty(), simpleConvention(FAIL_ON_NULL, BLOCK_POSITION, BLOCK_POSITION)); + OperatorConvention comparisonOperator = new OperatorConvention( + operatorConvention.type(), + comparisonType, + Optional.empty(), + // null positions are handled separately in adaptBlockPositionComparisonToOrdering and not used in the comparison + simpleConvention(FAIL_ON_NULL, BLOCK_POSITION_NOT_NULL, BLOCK_POSITION_NOT_NULL)); MethodHandle comparisonInvoker = adaptOperator(comparisonOperator); return adaptBlockPositionComparisonToOrdering(sortOrder, comparisonInvoker); } @@ -538,7 +544,7 @@ private static int getScore(OperatorMethodHandle operatorMethodHandle) if (argument == NULL_FLAG || argument == FLAT) { score += 100; } - else if (argument == BLOCK_POSITION) { + else if (argument == BLOCK_POSITION || argument == VALUE_BLOCK_POSITION_NOT_NULL) { score += 1; } } diff --git a/core/trino-spi/src/main/java/io/trino/spi/type/UuidType.java b/core/trino-spi/src/main/java/io/trino/spi/type/UuidType.java index e745dfab294a..25ea659d6730 100644 --- a/core/trino-spi/src/main/java/io/trino/spi/type/UuidType.java +++ b/core/trino-spi/src/main/java/io/trino/spi/type/UuidType.java @@ -123,19 +123,6 @@ public Object getObjectValue(Block block, int position) return new UUID(high, low).toString(); } - @Override - public void appendTo(Block block, int position, BlockBuilder blockBuilder) - { - if (block.isNull(position)) { - blockBuilder.appendNull(); - } - else { - Int128ArrayBlock valueBlock = (Int128ArrayBlock) block.getUnderlyingValueBlock(); - int valuePosition = block.getUnderlyingValuePosition(position); - ((Int128ArrayBlockBuilder) blockBuilder).writeInt128(valueBlock.getInt128High(valuePosition), valueBlock.getInt128Low(valuePosition)); - } - } - @Override public void writeSlice(BlockBuilder blockBuilder, Slice value) { diff --git a/core/trino-spi/src/test/java/io/trino/spi/block/AbstractTestBlockBuilder.java b/core/trino-spi/src/test/java/io/trino/spi/block/AbstractTestBlockBuilder.java index ceca56bae085..28ec61b07cb5 100644 --- a/core/trino-spi/src/test/java/io/trino/spi/block/AbstractTestBlockBuilder.java +++ b/core/trino-spi/src/test/java/io/trino/spi/block/AbstractTestBlockBuilder.java @@ -24,6 +24,8 @@ public abstract class AbstractTestBlockBuilder { + private static final int[] OFFSETS = new int[] {0, 2}; + protected abstract BlockBuilder createBlockBuilder(); protected abstract List getTestValues(); @@ -52,197 +54,216 @@ public void verifyTestData() public void testAppend() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlock(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlock(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.append(inputValues, 1); - blockBuilder.append(inputValues, 3); - blockBuilder.append(inputValues, 1); - blockBuilder.append(inputValues, 3); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.append(inputValues, 1); + blockBuilder.append(inputValues, 3); + blockBuilder.append(inputValues, 1); + blockBuilder.append(inputValues, 3); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isFalse(); + assertThat(valueBlock.mayHaveNull()).isFalse(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactly(values.get(1), values.get(3), values.get(1), values.get(3)); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactly(values.get(1), values.get(3), values.get(1), values.get(3)); + } } @Test public void testAppendWithNulls() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.append(inputValues, 1); - blockBuilder.append(inputValues, 3); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.append(inputValues, 1); + blockBuilder.append(inputValues, 3); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isTrue(); + assertThat(valueBlock.mayHaveNull()).isTrue(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).hasSize(2).containsOnlyNulls(); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).hasSize(2).containsOnlyNulls(); - // add a non-null value - blockBuilder.append(inputValues, 2); - valueBlock = blockBuilder.buildValueBlock(); + // add a non-null value + blockBuilder.append(inputValues, 2); + valueBlock = blockBuilder.buildValueBlock(); - actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactly(null, null, values.get(2)); + actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactly(null, null, values.get(2)); + } } @Test public void testAppendRepeated() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlock(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlock(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.appendRepeated(inputValues, 1, 10); - blockBuilder.appendRepeated(inputValues, 3, 10); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.appendRepeated(inputValues, 1, 10); + blockBuilder.appendRepeated(inputValues, 3, 10); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isFalse(); + assertThat(valueBlock.mayHaveNull()).isFalse(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactlyElementsOf(Iterables.concat(nCopies(10, values.get(1)), nCopies(10, values.get(3)))); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactlyElementsOf(Iterables.concat(nCopies(10, values.get(1)), nCopies(10, values.get(3)))); + } } @Test public void testAppendRepeatedWithNulls() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.appendRepeated(inputValues, 1, 10); - blockBuilder.appendRepeated(inputValues, 3, 10); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.appendRepeated(inputValues, 1, 10); + blockBuilder.appendRepeated(inputValues, 3, 10); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isTrue(); + assertThat(valueBlock.mayHaveNull()).isTrue(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).hasSize(20).containsOnlyNulls(); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).hasSize(20).containsOnlyNulls(); - // an all-null block should be converted to a RunLengthEncodedBlock - assertThat(blockBuilder.build()).isInstanceOf(RunLengthEncodedBlock.class); + // an all-null block should be converted to a RunLengthEncodedBlock + assertThat(blockBuilder.build()).isInstanceOf(RunLengthEncodedBlock.class); - // add some non-null values - blockBuilder.appendRepeated(inputValues, 2, 10); - valueBlock = blockBuilder.buildValueBlock(); + // add some non-null values + blockBuilder.appendRepeated(inputValues, 2, 10); + valueBlock = blockBuilder.buildValueBlock(); - actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactlyElementsOf(Iterables.concat(nCopies(20, null), nCopies(10, values.get(2)))); + actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactlyElementsOf(Iterables.concat(nCopies(20, null), nCopies(10, values.get(2)))); + } } @Test public void testAppendRange() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlock(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlock(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.appendRange(inputValues, 1, 3); - blockBuilder.appendRange(inputValues, 2, 3); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.appendRange(inputValues, 1, 3); + blockBuilder.appendRange(inputValues, 2, 3); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isFalse(); + assertThat(valueBlock.mayHaveNull()).isFalse(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactlyElementsOf(Iterables.concat(values.subList(1, 4), values.subList(2, 5))); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactlyElementsOf(Iterables.concat(values.subList(1, 4), values.subList(2, 5))); + } } @Test public void testAppendRangeWithNulls() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.appendRange(inputValues, 1, 3); - blockBuilder.appendRange(inputValues, 2, 3); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.appendRange(inputValues, 1, 3); + blockBuilder.appendRange(inputValues, 2, 3); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isTrue(); + assertThat(valueBlock.mayHaveNull()).isTrue(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactly(null, values.get(2), null, values.get(2), null, values.get(4)); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactly(null, values.get(2), null, values.get(2), null, values.get(4)); + } } @Test public void testAppendPositions() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlock(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlock(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.appendPositions(inputValues, new int[] {-100, 1, 3, 2, -100}, 1, 3); - blockBuilder.appendPositions(inputValues, new int[] {-100, 4, 0, -100}, 1, 2); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.appendPositions(inputValues, new int[] {-100, 1, 3, 2, -100}, 1, 3); + blockBuilder.appendPositions(inputValues, new int[] {-100, 4, 0, -100}, 1, 2); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isFalse(); + assertThat(valueBlock.mayHaveNull()).isFalse(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactly(values.get(1), values.get(3), values.get(2), values.get(4), values.get(0)); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactly(values.get(1), values.get(3), values.get(2), values.get(4), values.get(0)); + } } @Test public void testAppendPositionsWithNull() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlockWithOddPositionsNull(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.appendPositions(inputValues, new int[] {-100, 1, 3, 2, -100}, 1, 3); - blockBuilder.appendPositions(inputValues, new int[] {-100, 4, 0, -100}, 1, 2); - ValueBlock valueBlock = blockBuilder.buildValueBlock(); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.appendPositions(inputValues, new int[] {-100, 1, 3, 2, -100}, 1, 3); + blockBuilder.appendPositions(inputValues, new int[] {-100, 4, 0, -100}, 1, 2); + ValueBlock valueBlock = blockBuilder.buildValueBlock(); - assertThat(valueBlock.mayHaveNull()).isTrue(); + assertThat(valueBlock.mayHaveNull()).isTrue(); - List actualValues = blockToValues(valueBlock); - assertThat(actualValues).containsExactly(null, null, values.get(2), values.get(4), values.get(0)); + List actualValues = blockToValues(valueBlock); + assertThat(actualValues).containsExactly(null, null, values.get(2), values.get(4), values.get(0)); + } } @Test public void testResetTo() { List values = getTestValues(); - ValueBlock inputValues = createOffsetBlock(values); + for (int offset : OFFSETS) { + ValueBlock inputValues = createOffsetBlock(values, offset); - BlockBuilder blockBuilder = createBlockBuilder(); - blockBuilder.appendRange(inputValues, 0, inputValues.getPositionCount()); - assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(values); + BlockBuilder blockBuilder = createBlockBuilder(); + blockBuilder.appendRange(inputValues, 0, inputValues.getPositionCount()); + assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(values); - blockBuilder.resetTo(4); - assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(values.subList(0, 4)); + blockBuilder.resetTo(4); + assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(values.subList(0, 4)); - blockBuilder.appendRange(inputValues, 0, inputValues.getPositionCount()); - assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(Iterables.concat(values.subList(0, 4), values)); + blockBuilder.appendRange(inputValues, 0, inputValues.getPositionCount()); + assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(Iterables.concat(values.subList(0, 4), values)); - blockBuilder.resetTo(0); - assertThat(blockToValues(blockBuilder.buildValueBlock())).isEmpty(); + blockBuilder.resetTo(0); + assertThat(blockToValues(blockBuilder.buildValueBlock())).isEmpty(); - blockBuilder.appendRange(inputValues, 0, inputValues.getPositionCount()); - assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(values); + blockBuilder.appendRange(inputValues, 0, inputValues.getPositionCount()); + assertThat(blockToValues(blockBuilder.buildValueBlock())).containsExactlyElementsOf(values); + } } /** * Create a block that is offset from the start of the underlying array */ - private ValueBlock createOffsetBlock(List values) + private ValueBlock createOffsetBlock(List values, int offset) { - return blockFromValues(Iterables.concat(nCopies(2, getUnusedTestValue()), values, nCopies(2, getUnusedTestValue()))) - .getRegion(2, values.size()); + return blockFromValues(Iterables.concat(nCopies(offset, getUnusedTestValue()), values, nCopies(offset, getUnusedTestValue()))) + .getRegion(offset, values.size()); } /** * Create a block that is offset from the start of the underlying array */ - private ValueBlock createOffsetBlockWithOddPositionsNull(List values) + private ValueBlock createOffsetBlockWithOddPositionsNull(List values, int offset) { ArrayList blockValues = new ArrayList<>(); - blockValues.add(getUnusedTestValue()); - blockValues.add(getUnusedTestValue()); + for (int i = 0; i < offset; i++) { + blockValues.add(getUnusedTestValue()); + } for (int i = 0; i < values.size(); i++) { T value = values.get(i); if (i % 2 == 0) { @@ -252,8 +273,9 @@ private ValueBlock createOffsetBlockWithOddPositionsNull(List values) blockValues.add(null); } } - blockValues.add(getUnusedTestValue()); - blockValues.add(getUnusedTestValue()); - return blockFromValues(blockValues).getRegion(2, values.size()); + for (int i = 0; i < offset; i++) { + blockValues.add(getUnusedTestValue()); + } + return blockFromValues(blockValues).getRegion(offset, values.size()); } } diff --git a/core/trino-web-ui/pom.xml b/core/trino-web-ui/pom.xml index 3247faf43783..1d94e1ef7633 100644 --- a/core/trino-web-ui/pom.xml +++ b/core/trino-web-ui/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/package-lock.json b/core/trino-web-ui/src/main/resources/webapp-preview/package-lock.json index 2b7be5830376..5c734bd5166c 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/package-lock.json +++ b/core/trino-web-ui/src/main/resources/webapp-preview/package-lock.json @@ -12,6 +12,7 @@ "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", "@fontsource/roboto": "^5.2.8", + "@monaco-editor/react": "^4.7.0", "@mui/icons-material": "^7.3.2", "@mui/material": "^7.3.2", "@mui/material-pigment-css": "^7.3.2", @@ -23,16 +24,15 @@ "react-calendar-timeline": "^0.30.0-beta.3", "react-dom": "^19.1.1", "react-router-dom": "^7.9.1", - "react-syntax-highlighter": "^15.6.6", "sass": "^1.93.2", "zustand": "^5.0.8" }, "devDependencies": { "@eslint/js": "^9.36.0", "@types/lodash": "^4.17.20", + "@types/lodash.merge": "^4.6.9", "@types/react": "^19.1.13", "@types/react-dom": "^19.1.9", - "@types/react-syntax-highlighter": "^15.5.13", "@typescript-eslint/eslint-plugin": "^8.44.1", "@typescript-eslint/parser": "^8.44.1", "@vitejs/plugin-react": "^5.0.3", @@ -1276,6 +1276,29 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@monaco-editor/loader": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@monaco-editor/loader/-/loader-1.5.0.tgz", + "integrity": "sha512-hKoGSM+7aAc7eRTRjpqAZucPmoNOC4UUbknb/VNoTkEIkCPhqV8LfbsgM1webRM7S/z21eHEx9Fkwx8Z/C/+Xw==", + "license": "MIT", + "dependencies": { + "state-local": "^1.0.6" + } + }, + "node_modules/@monaco-editor/react": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/@monaco-editor/react/-/react-4.7.0.tgz", + "integrity": "sha512-cyzXQCtO47ydzxpQtCGSQGOC8Gk3ZUeBXFAxD+CWXYFo5OqZyZUonFl0DwUlTyAfRHntBfw2p3w4s9R6oe1eCA==", + "license": "MIT", + "dependencies": { + "@monaco-editor/loader": "^1.5.0" + }, + "peerDependencies": { + "monaco-editor": ">= 0.25.0 < 1", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@mui/core-downloads-tracker": { "version": "7.3.2", "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-7.3.2.tgz", @@ -2648,15 +2671,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/hast": { - "version": "2.3.10", - "resolved": "https://registry.npmjs.org/@types/hast/-/hast-2.3.10.tgz", - "integrity": "sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==", - "license": "MIT", - "dependencies": { - "@types/unist": "^2" - } - }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -2671,6 +2685,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/lodash.merge": { + "version": "4.6.9", + "resolved": "https://registry.npmjs.org/@types/lodash.merge/-/lodash.merge-4.6.9.tgz", + "integrity": "sha512-23sHDPmzd59kUgWyKGiOMO2Qb9YtqRO/x4IhkgNUiPQ1+5MUVqi6bCZeq9nBJ17msjIMbEIO5u+XW4Kz6aGUhQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/lodash": "*" + } + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -2702,16 +2726,6 @@ "@types/react": "^19.0.0" } }, - "node_modules/@types/react-syntax-highlighter": { - "version": "15.5.13", - "resolved": "https://registry.npmjs.org/@types/react-syntax-highlighter/-/react-syntax-highlighter-15.5.13.tgz", - "integrity": "sha512-uLGJ87j6Sz8UaBAooU0T6lWJ0dBmjZgN1PZTrj05TNql2/XpC6+4HhMT5syIdFUUt+FASfCeLLv4kBygNU+8qA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/react": "*" - } - }, "node_modules/@types/react-transition-group": { "version": "4.4.12", "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.12.tgz", @@ -2721,11 +2735,12 @@ "@types/react": "*" } }, - "node_modules/@types/unist": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", - "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", - "license": "MIT" + "node_modules/@types/trusted-types": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-1.0.6.tgz", + "integrity": "sha512-230RC8sFeHoT6sSUlRO6a8cAnclO06eeiq1QDfiv2FGCLWFvvERWgwIQD4FWqD9A69BN7Lzee4OXwoMVnnsWDw==", + "license": "MIT", + "peer": true }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.44.1", @@ -3604,36 +3619,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/character-entities": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", - "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities-legacy": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", - "integrity": "sha512-3Xnr+7ZFS1uxeiUDvV02wQ+QDbc55o97tIV5zHScSPJpcLm/r0DFPcoY3tYRp+VZukxuMeKgXYmsXQHO05zQeA==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-reference-invalid": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.4.tgz", - "integrity": "sha512-mKKUkUbhPpQlCOfIuZkvSEgktjPFIsZKRRbC6KWVEMvlzblj3i3asQv5ODsrwt0N3pHAEvjP8KTQPHkp0+6jOg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/chokidar": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", @@ -3702,16 +3687,6 @@ "node": ">= 0.8" } }, - "node_modules/comma-separated-tokens": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-1.0.8.tgz", - "integrity": "sha512-GHuDRO12Sypu2cV70d1dkA2EUmXHgntrzbpvOB+Qy+49ypNfGgFQIC2fhhXbnyrJRynDCAARsT7Ou0M6hirpfw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -4872,19 +4847,6 @@ "reusify": "^1.0.4" } }, - "node_modules/fault": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", - "integrity": "sha512-CJ0HCB5tL5fYTEA7ToAq5+kTwd++Borf1/bifxd9iT70QcXr4MRrO3Llf8Ifs70q+SJcGHFtnIE/Nw6giCtECA==", - "license": "MIT", - "dependencies": { - "format": "^0.2.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -5006,14 +4968,6 @@ "node": ">= 6" } }, - "node_modules/format": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", - "integrity": "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==", - "engines": { - "node": ">=0.4.x" - } - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -5301,48 +5255,6 @@ "node": ">= 0.4" } }, - "node_modules/hast-util-parse-selector": { - "version": "2.2.5", - "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", - "integrity": "sha512-7j6mrk/qqkSehsM92wQjdIgWM2/BW61u/53G6xmC8i1OmEdKLHbk419QKQUjz6LglWsfqoiHmyMRkP1BGjecNQ==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/hastscript": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-6.0.0.tgz", - "integrity": "sha512-nDM6bvd7lIqDUiYEiu5Sl/+6ReP0BMk/2f4U/Rooccxkj0P5nm+acM5PrGJ/t5I8qPGiqZSE6hVAwZEdZIvP4w==", - "license": "MIT", - "dependencies": { - "@types/hast": "^2.0.0", - "comma-separated-tokens": "^1.0.0", - "hast-util-parse-selector": "^2.0.0", - "property-information": "^5.0.0", - "space-separated-tokens": "^1.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/highlight.js": { - "version": "10.7.3", - "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", - "integrity": "sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "node_modules/highlightjs-vue": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/highlightjs-vue/-/highlightjs-vue-1.0.0.tgz", - "integrity": "sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA==", - "license": "CC0-1.0" - }, "node_modules/hoist-non-react-statics": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", @@ -5431,30 +5343,6 @@ "integrity": "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==", "license": "ISC" }, - "node_modules/is-alphabetical": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", - "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/is-alphanumerical": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz", - "integrity": "sha512-UzoZUr+XfVz3t3v4KyGEniVL9BDRoQtY7tOyrRybkVNjDFWyo1yhXNGrrBTQxp3ib9BLAWs7k2YKBQsFRkZG9A==", - "license": "MIT", - "dependencies": { - "is-alphabetical": "^1.0.0", - "is-decimal": "^1.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/is-array-buffer": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.5.tgz", @@ -5595,16 +5483,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-decimal": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.4.tgz", - "integrity": "sha512-RGdriMmQQvZ2aqaQq3awNA6dCGtKpiDFcOzrTWrDAT2MiWrKQVPmxLGHl7Y2nNu6led0kEyoX0enY0qXYsv9zw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", @@ -5676,16 +5554,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-hexadecimal": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.4.tgz", - "integrity": "sha512-gyPJuv83bHMpocVYoqof5VDiZveEoGoFL8m3BXNb2VW8Xs+rz9kqO8LOQ5DH6EsuvilT1ApazU0pyl+ytbPtlw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/is-map": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.3.tgz", @@ -6094,20 +5962,6 @@ "loose-envify": "cli.js" } }, - "node_modules/lowlight": { - "version": "1.20.0", - "resolved": "https://registry.npmjs.org/lowlight/-/lowlight-1.20.0.tgz", - "integrity": "sha512-8Ktj+prEb1RoCPkEOrPMYUN/nCggB7qAWe3a7OpMjWQkh3l2RD5wKRQ+o8Q8YuI9RG/xs95waaI/E6ym/7NsTw==", - "license": "MIT", - "dependencies": { - "fault": "^1.0.0", - "highlight.js": "~10.7.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -6192,6 +6046,16 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/monaco-editor": { + "version": "0.53.0", + "resolved": "https://registry.npmjs.org/monaco-editor/-/monaco-editor-0.53.0.tgz", + "integrity": "sha512-0WNThgC6CMWNXXBxTbaYYcunj08iB5rnx4/G56UOPeL9UVIUGGHA1GR0EWIh9Ebabj7NpCRawQ5b0hfN1jQmYQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/trusted-types": "^1.0.6" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -6435,24 +6299,6 @@ "node": ">=6" } }, - "node_modules/parse-entities": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", - "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", - "license": "MIT", - "dependencies": { - "character-entities": "^1.0.0", - "character-entities-legacy": "^1.0.0", - "character-reference-invalid": "^1.0.0", - "is-alphanumerical": "^1.0.0", - "is-decimal": "^1.0.0", - "is-hexadecimal": "^1.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", @@ -6589,15 +6435,6 @@ "url": "https://github.com/prettier/prettier?sponsor=1" } }, - "node_modules/prismjs": { - "version": "1.30.0", - "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", - "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", @@ -6615,19 +6452,6 @@ "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "license": "MIT" }, - "node_modules/property-information": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz", - "integrity": "sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -6758,23 +6582,6 @@ "react-dom": ">=18" } }, - "node_modules/react-syntax-highlighter": { - "version": "15.6.6", - "resolved": "https://registry.npmjs.org/react-syntax-highlighter/-/react-syntax-highlighter-15.6.6.tgz", - "integrity": "sha512-DgXrc+AZF47+HvAPEmn7Ua/1p10jNoVZVI/LoPiYdtY+OM+/nG5yefLHKJwdKqY1adMuHFbeyBaG9j64ML7vTw==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.3.1", - "highlight.js": "^10.4.1", - "highlightjs-vue": "^1.0.0", - "lowlight": "^1.17.0", - "prismjs": "^1.30.0", - "refractor": "^3.6.0" - }, - "peerDependencies": { - "react": ">= 0.14.0" - } - }, "node_modules/react-transition-group": { "version": "4.4.5", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", @@ -6827,21 +6634,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/refractor": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", - "integrity": "sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA==", - "license": "MIT", - "dependencies": { - "hastscript": "^6.0.0", - "parse-entities": "^2.0.0", - "prismjs": "~1.27.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/regexp.prototype.flags": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", @@ -7243,15 +7035,11 @@ "node": ">=0.10.0" } }, - "node_modules/space-separated-tokens": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-1.1.5.tgz", - "integrity": "sha512-q/JSVd1Lptzhf5bkYm4ob4iWPjx0KiRe3sRFBNrVqbJkFaBm5vbbowy1mymoPNLRa52+oadOhJ+K49wsSeSjTA==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } + "node_modules/state-local": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/state-local/-/state-local-1.0.7.tgz", + "integrity": "sha512-HTEHMNieakEnoe33shBYcZ7NX83ACUjCu8c40iOGEZsngj9zRnkqS9j1pqQPXwobB0ZcVTk27REb7COQ0UR59w==", + "license": "MIT" }, "node_modules/stop-iteration-iterator": { "version": "1.1.0", @@ -7954,15 +7742,6 @@ "node": ">=0.10.0" } }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/package.json b/core/trino-web-ui/src/main/resources/webapp-preview/package.json index 30d6781e5509..15c9a01ac283 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/package.json +++ b/core/trino-web-ui/src/main/resources/webapp-preview/package.json @@ -20,6 +20,7 @@ "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.1", "@fontsource/roboto": "^5.2.8", + "@monaco-editor/react": "^4.7.0", "@mui/icons-material": "^7.3.2", "@mui/material": "^7.3.2", "@mui/material-pigment-css": "^7.3.2", @@ -31,16 +32,15 @@ "react-calendar-timeline": "^0.30.0-beta.3", "react-dom": "^19.1.1", "react-router-dom": "^7.9.1", - "react-syntax-highlighter": "^15.6.6", "sass": "^1.93.2", "zustand": "^5.0.8" }, "devDependencies": { "@eslint/js": "^9.36.0", "@types/lodash": "^4.17.20", + "@types/lodash.merge": "^4.6.9", "@types/react": "^19.1.13", "@types/react-dom": "^19.1.9", - "@types/react-syntax-highlighter": "^15.5.13", "@typescript-eslint/eslint-plugin": "^8.44.1", "@typescript-eslint/parser": "^8.44.1", "@vitejs/plugin-react": "^5.0.3", diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/api/webapp/api.ts b/core/trino-web-ui/src/main/resources/webapp-preview/src/api/webapp/api.ts index c94fca1560b8..51d4b37e3444 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/api/webapp/api.ts +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/api/webapp/api.ts @@ -175,6 +175,7 @@ export interface QueryInfo extends QueryInfoBase { sessionSource: string sessionUser: string queryDataEncoding: string + traceToken: string } export interface Session { diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/CodeBlock.tsx b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/CodeBlock.tsx index 8d5ddb767713..5427fdb40c12 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/CodeBlock.tsx +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/CodeBlock.tsx @@ -11,31 +11,76 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { useState } from 'react' import { Box, useMediaQuery } from '@mui/material' -import { LightAsync as SyntaxHighlighter } from 'react-syntax-highlighter' -import { a11yLight, a11yDark } from 'react-syntax-highlighter/dist/esm/styles/hljs' +import Editor, { type OnMount } from '@monaco-editor/react' +import merge from 'lodash.merge' import { Theme as ThemeStore, useConfigStore } from '../store' +import type { editor as MonacoEditor } from 'monaco-editor' + +const FALLBACK_LINE_HEIGHT_PX = 20 +const EDITOR_EXTRA_PADDING = 4 +const VIEWPORT_HEIGHT_RATIO = 0.75 +const LINE_BREAK_REGEX = /\r\n|\r|\n/ export interface ICodeBlockProps { code: string language: string height?: string noBottomBorder?: boolean + monacoOptions?: MonacoEditor.IStandaloneEditorConstructionOptions } export const CodeBlock = (props: ICodeBlockProps) => { const config = useConfigStore() - const { code, language, height, noBottomBorder } = props + const { code, language, height, noBottomBorder, monacoOptions } = props const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)') + const [resolvedHeight, setResolvedHeight] = useState(height) - const styleToUse = () => { + const themeToUse = () => { if (config.theme === ThemeStore.Auto) { - return prefersDarkMode ? a11yDark : a11yLight + return prefersDarkMode ? 'vs-dark' : 'light' } else if (config.theme === ThemeStore.Dark) { - return a11yDark + return 'vs-dark' } else { - return a11yLight + return 'light' + } + } + + // Calculates a dynamic editor height if none is provided. + // Uses the editor's lineHeight and code line count to estimate total height, + // adds extra padding, then clamps it so it never exceeds a fraction of the viewport. + // Falls back to a fixed height if one was passed in. + const handleEditorMount: OnMount = (editor, monaco) => { + if (height) { + setResolvedHeight(height) + return } + + const lineHeightOption = editor.getOption(monaco.editor.EditorOption.lineHeight) + const lineHeight = typeof lineHeightOption === 'number' ? lineHeightOption : FALLBACK_LINE_HEIGHT_PX + const lineCount = Math.max(code.split(LINE_BREAK_REGEX).length, 1) + const estimatedHeight = lineCount * lineHeight + EDITOR_EXTRA_PADDING + const viewportCap = window.innerHeight * VIEWPORT_HEIGHT_RATIO + const clampedHeight = Math.min(estimatedHeight, viewportCap) + setResolvedHeight(`${Math.round(clampedHeight)}px`) + } + + const defaultMonacoOptions: MonacoEditor.IStandaloneEditorConstructionOptions = { + readOnly: true, + domReadOnly: true, + minimap: { enabled: false }, + scrollBeyondLastLine: false, + lineNumbers: 'off', + folding: false, + contextmenu: false, + renderLineHighlight: 'none', + overviewRulerLanes: 0, + wordWrap: 'on', + rulers: [], + guides: { indentation: false }, + stickyScroll: { enabled: false }, + scrollBeyondLastColumn: 0, } return ( @@ -49,26 +94,17 @@ export const CodeBlock = (props: ICodeBlockProps) => { border: `1px solid ${theme.palette.mode === 'dark' ? '#3f3f3f' : '#ddd'}`, borderBottom: noBottomBorder ? 'none' : '', width: '100%', - height: { - xs: '100%', - lg: height, - }, + maxHeight: '90vh', })} > - - {code} - + theme={themeToUse()} + onMount={handleEditorMount} + value={code} + options={merge({}, defaultMonacoOptions, monacoOptions)} + /> ) } diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryJson.tsx b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryJson.tsx index 6ccf54b26d17..7fc677f0334b 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryJson.tsx +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryJson.tsx @@ -56,7 +56,17 @@ export const QueryJson = () => { - + diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryList.tsx b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryList.tsx index aebbf8e43c55..ff6162ba7fc9 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryList.tsx +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryList.tsx @@ -171,6 +171,7 @@ export const QueryList = () => { query.resourceGroupId?.join('.'), query.errorCode?.name, ...(query.clientTags || []), + query.traceToken, ].some((value) => value?.toLowerCase().includes(term)) }) } diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryLivePlan.tsx b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryLivePlan.tsx index 9cf1e2f00c9a..660511ce941d 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryLivePlan.tsx +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryLivePlan.tsx @@ -13,12 +13,12 @@ */ import { useParams } from 'react-router-dom' import { useEffect, useRef, useState } from 'react' -import { Alert, Box, CircularProgress, Grid } from '@mui/material' -import { ReactFlow, type Edge, type Node, useNodesState, useEdgesState } from '@xyflow/react' +import { Alert, Box, CircularProgress, Grid, Typography } from '@mui/material' +import { ReactFlow, type Edge, type Node, useNodesState, useEdgesState, type Viewport } from '@xyflow/react' import '@xyflow/react/dist/style.css' import { queryStatusApi, QueryStatusInfo } from '../api/webapp/api.ts' import { QueryProgressBar } from './QueryProgressBar' -import { nodeTypes, getLayoutedPlanFlowElements } from './flow/layout' +import { nodeTypes, getLayoutedPlanFlowElements, getViewportFocusedOnNode } from './flow/layout' import { HelpMessage } from './flow/HelpMessage' import { getPlanFlowElements } from './flow/flowUtils' import { IQueryStatus, LayoutDirectionType } from './flow/types' @@ -36,6 +36,7 @@ export const QueryLivePlan = () => { const [nodes, setNodes, onNodesChange] = useNodesState([]) const [edges, setEdges, onEdgesChange] = useEdgesState([]) const [layoutDirection, setLayoutDirection] = useState('BT') + const [viewport, setViewport] = useState() const [loading, setLoading] = useState(true) const [error, setError] = useState(null) @@ -94,6 +95,16 @@ export const QueryLivePlan = () => { } } + const focusViewportToFirstStage = () => { + const viewportTarget = getViewportFocusedOnNode(nodes, { + targetNodeId: 'stage-0', + containerWidth: containerRef.current?.clientWidth ?? 0, + }) + if (viewportTarget) { + setViewport(viewportTarget) + } + } + return ( <> {loading && } @@ -115,21 +126,30 @@ export const QueryLivePlan = () => { ref={containerRef} sx={{ width: '100%', height: '80vh', border: '1px solid #ccc' }} > - - - + {nodes.length > 0 ? ( + + + + ) : ( + + Rendering... + + )} diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QuerySplitsTimeline.tsx b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QuerySplitsTimeline.tsx index 63495d16c56e..948e83c1560a 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QuerySplitsTimeline.tsx +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QuerySplitsTimeline.tsx @@ -177,8 +177,11 @@ export const QuerySplitsTimeline = () => { return No split timeline data available. } - const timelineStartTime = items.reduce((min, item) => Math.min(min, item.start_time), items[0].start_time) - const timelineEndTime = items.reduce((max, item) => Math.max(max, item.end_time), items[0].end_time) + const startTimes = items.map((i) => i.start_time).filter(Number.isFinite) + const endTimes = items.map((i) => i.end_time).filter(Number.isFinite) + + const timelineStartTime = startTimes.length ? Math.min(...startTimes) : null + const timelineEndTime = endTimes.length ? Math.max(...endTimes) : null const itemRenderer: TimelineItemRenderer = ({ item, itemContext, getItemProps }: TimelineItemRendererProps) => { const splitItem = item as SplitTimelineItem @@ -286,20 +289,28 @@ export const QuerySplitsTimeline = () => { ))} - + {timelineStartTime && timelineEndTime ? ( + + ) : ( + + + Splits timeline will appear automatically when at least one query task starts running + + + )} ) } diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryStagePerformance.tsx b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryStagePerformance.tsx index 442d27fd8cbc..b8bcbb43a74c 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryStagePerformance.tsx +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/QueryStagePerformance.tsx @@ -24,13 +24,13 @@ import { Select, SelectChangeEvent, } from '@mui/material' -import { type Edge, type Node, ReactFlow, useEdgesState, useNodesState } from '@xyflow/react' +import { type Edge, type Node, ReactFlow, useEdgesState, useNodesState, type Viewport } from '@xyflow/react' import { queryStatusApi, QueryStatusInfo, QueryStage } from '../api/webapp/api.ts' import { ApiResponse } from '../api/base.ts' import { Texts } from '../constant.ts' import { QueryProgressBar } from './QueryProgressBar.tsx' import { HelpMessage } from './flow/HelpMessage' -import { nodeTypes, getLayoutedStagePerformanceElements } from './flow/layout' +import { nodeTypes, getLayoutedStagePerformanceElements, getViewportFocusedOnNode } from './flow/layout' import { LayoutDirectionType } from './flow/types' import { getStagePerformanceFlowElements } from './flow/flowUtils.ts' @@ -53,6 +53,7 @@ export const QueryStagePerformance = () => { const [nodes, setNodes, onNodesChange] = useNodesState([]) const [edges, setEdges, onEdgesChange] = useEdgesState([]) const [layoutDirection, setLayoutDirection] = useState('BT') + const [viewport, setViewport] = useState() const [loading, setLoading] = useState(true) const [error, setError] = useState(null) @@ -135,6 +136,16 @@ export const QueryStagePerformance = () => { }, } + const focusViewportToFirstPipeline = () => { + const viewportTarget = getViewportFocusedOnNode(nodes, { + targetNodeId: 'pipeline-0', + containerWidth: containerRef.current?.clientWidth ?? 0, + }) + if (viewportTarget) { + setViewport(viewportTarget) + } + } + const handleStageIdChange = (event: SelectChangeEvent) => { setStagePlanId(event.target.value as string) } @@ -169,11 +180,14 @@ export const QueryStagePerformance = () => { nodeTypes={nodeTypes} minZoom={0.1} proOptions={{ hideAttribution: true }} - defaultViewport={{ x: 200, y: 20, zoom: 0.8 }} + viewport={viewport} + onViewportChange={setViewport} + fitView > diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/HelpMessage.tsx b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/HelpMessage.tsx index ff9aecbd695f..c919b5d79e3e 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/HelpMessage.tsx +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/HelpMessage.tsx @@ -11,23 +11,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Box, Typography, ToggleButtonGroup, ToggleButton } from '@mui/material' +import { Box, Button, Typography, ToggleButtonGroup, ToggleButton } from '@mui/material' import React from 'react' import { LayoutDirectionType } from './types' interface IHelpMessageProps { layoutDirection: LayoutDirectionType onLayoutDirectionChange: (layoutDirection: LayoutDirectionType) => void + onOriginClick: () => void additionalContent?: React.ReactNode } -export const HelpMessage = ({ layoutDirection, onLayoutDirectionChange, additionalContent }: IHelpMessageProps) => { +export const HelpMessage = ({ + layoutDirection, + onLayoutDirectionChange, + onOriginClick, + additionalContent, +}: IHelpMessageProps) => { const handleLayoutChange = (_event: React.MouseEvent, newDirection: LayoutDirectionType | null) => { if (newDirection !== null) { onLayoutDirectionChange(newDirection) } } + const handleOriginClick = () => { + onOriginClick() + } + return ( - - Scroll to zoom in/out - + + + Scroll to zoom in/out + + + Horizontal - {additionalContent} ) diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/layout.ts b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/layout.ts index 73dfb7d25da4..bf43c376d960 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/layout.ts +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/components/flow/layout.ts @@ -12,7 +12,7 @@ * limitations under the License. */ import Dagre from '@dagrejs/dagre' -import { type Edge, type Node } from '@xyflow/react' +import { type Edge, type Node, type Viewport } from '@xyflow/react' import { RemoteExchangeNode } from './RemoteExchangeNode.tsx' import { PlanFragmentNode } from './PlanFragmentNode.tsx' import { OperatorNode } from './OperatorNode.tsx' @@ -184,3 +184,24 @@ export const getLayoutedStagePerformanceElements = (nodes: Node[], edges: Edge[] edges, } } + +export const getViewportFocusedOnNode = ( + nodes: Node[], + options?: { targetNodeId?: string; zoom?: number; padding?: number; containerWidth?: number } +): Viewport | undefined => { + if (nodes.length === 0) { + return undefined + } + const { targetNodeId, zoom = 0.8, padding = 20, containerWidth = 0 } = options ?? {} + const targetNode = targetNodeId ? nodes.find((node) => node.id === targetNodeId) : undefined + const focusNode = targetNode ?? nodes[0] + const focusNodeWidth = focusNode.measured?.width ?? focusNode.width ?? 0 + return { + // If width is known then center horizontally in container otherwise align to left + x: focusNodeWidth + ? containerWidth / 2 - (focusNode.position.x + focusNodeWidth / 2) * zoom + : -focusNode.position.x * zoom + padding, + y: -focusNode.position.y * zoom + padding, + zoom, + } +} diff --git a/core/trino-web-ui/src/main/resources/webapp-preview/src/constant.ts b/core/trino-web-ui/src/main/resources/webapp-preview/src/constant.ts index 08f7fa7d77df..d824abe5d8b4 100644 --- a/core/trino-web-ui/src/main/resources/webapp-preview/src/constant.ts +++ b/core/trino-web-ui/src/main/resources/webapp-preview/src/constant.ts @@ -68,7 +68,7 @@ export const Texts = { Filter: { Search: 'Search', SearchPlaceholder: - 'User, source, query ID, query state, resource group, error name, query text or client tags', + 'User, source, query ID, state, resource group, error name, query text, client tags or trace token', State: 'State', Type: { RUNNING: 'Running', diff --git a/core/trino-web-ui/src/main/resources/webapp/src/components/QueryList.jsx b/core/trino-web-ui/src/main/resources/webapp/src/components/QueryList.jsx index 816fd34b1b8d..533fe3c1429c 100644 --- a/core/trino-web-ui/src/main/resources/webapp/src/components/QueryList.jsx +++ b/core/trino-web-ui/src/main/resources/webapp/src/components/QueryList.jsx @@ -477,6 +477,10 @@ export class QueryList extends React.Component { ) { return true } + + if (query.traceToken && query.traceToken.toLowerCase().indexOf(term) !== -1) { + return true + } }, this) } } @@ -817,7 +821,7 @@ export class QueryList extends React.Component { diff --git a/docs/pom.xml b/docs/pom.xml index 12035027cd8d..a4a4264a0c5e 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT trino-docs @@ -51,7 +51,7 @@ generate-thrift-idl - validate + generate-sources ${project.build.directory}/TrinoThriftService.thrift @@ -83,7 +83,7 @@ java - validate + test io.trino.sql.ReservedIdentifiers @@ -97,7 +97,7 @@ exec - validate + test diff diff --git a/docs/src/main/sphinx/admin/fault-tolerant-execution.md b/docs/src/main/sphinx/admin/fault-tolerant-execution.md index a857d139980f..528ed5ce9c43 100644 --- a/docs/src/main/sphinx/admin/fault-tolerant-execution.md +++ b/docs/src/main/sphinx/admin/fault-tolerant-execution.md @@ -71,6 +71,11 @@ execution on a Trino cluster: fault-tolerant execution and typically only to deactivate with `NONE`, since switching between modes on a cluster is not tested. - `NONE` +* - `retry-policy.allowed` + - List of retry policies that are allowed to be configured for a cluster. + This property is used to prevent a user from configuring a retry policy that + is not meant to be used on the given cluster. + - `NONE`, `QUERY`, `TASK` * - `exchange.deduplication-buffer-size` - [Data size](prop-type-data-size) of the coordinator's in-memory buffer used by fault-tolerant execution to store output of query diff --git a/docs/src/main/sphinx/admin/resource-groups.md b/docs/src/main/sphinx/admin/resource-groups.md index ff408d4eaa26..8f0990587bf3 100644 --- a/docs/src/main/sphinx/admin/resource-groups.md +++ b/docs/src/main/sphinx/admin/resource-groups.md @@ -199,7 +199,11 @@ documentation](https://docs.oracle.com/en/java/javase/24/docs/api/java.base/java - `DATA_DEFINITION`: Queries that affect the data definition. These include `CREATE`, `ALTER`, and `DROP` statements for schemas, tables, views, and materialized views, as well as statements that manage prepared statements, - privileges, sessions, and transactions. + privileges, sessions, and transactions. When external clients need + access to the `system.runtime.kill_query()` procedure to stop running or + queued queries, this `queryType` must be used to make sure the + `kill_query()` is executed directly and isn't queued to wait for the + initial query to finish. - `ALTER_TABLE_EXECUTE`: Queries that execute table procedures with [ALTER TABLE EXECUTE](alter-table-execute). diff --git a/docs/src/main/sphinx/client/cli.md b/docs/src/main/sphinx/client/cli.md index 9df5771b46d7..d70341af0012 100644 --- a/docs/src/main/sphinx/client/cli.md +++ b/docs/src/main/sphinx/client/cli.md @@ -25,7 +25,7 @@ Versions before 350 are not supported. (cli-installation)= ## Installation -Download {maven_download}`cli`, rename it to `trino`, make it executable with +Download {download_gh}`cli`, rename it to `trino`, make it executable with `chmod +x`, and run it to show the version of the CLI: ```text diff --git a/docs/src/main/sphinx/client/jdbc.md b/docs/src/main/sphinx/client/jdbc.md index 34e113c7edf3..0058ef72d895 100644 --- a/docs/src/main/sphinx/client/jdbc.md +++ b/docs/src/main/sphinx/client/jdbc.md @@ -27,7 +27,7 @@ Versions before 350 are not supported. (jdbc-installation)= ## Installation -Download {maven_download}`jdbc` and add it to the classpath of your Java application. +Download {download_mc}`jdbc` and add it to the classpath of your Java application. The driver is also available from Maven Central: @@ -248,7 +248,10 @@ may not be specified using both methods. different users, the first registered token is stored and authenticates all users. * - `disableCompression` - - Whether compression should be enabled. + - Whether HTTP compression should be disabled. Defaults to `false`. +* - `disallowLocalRedirect` + - Whether client should reject redirects to localhost, link or site local + IP addresses. Defaults to `false`. * - `assumeLiteralUnderscoreInMetadataCallsForNonConformingClients` - When enabled, the name patterns passed to `DatabaseMetaData` methods are treated as underscores. You can use this as a workaround for applications diff --git a/docs/src/main/sphinx/connector.md b/docs/src/main/sphinx/connector.md index a954b30cf059..1a77556400c3 100644 --- a/docs/src/main/sphinx/connector.md +++ b/docs/src/main/sphinx/connector.md @@ -42,6 +42,7 @@ SingleStore Snowflake SQL Server System +Teradata Thrift TPC-DS TPC-H diff --git a/docs/src/main/sphinx/connector/delta-lake.md b/docs/src/main/sphinx/connector/delta-lake.md index 288b538617a3..858c762cf341 100644 --- a/docs/src/main/sphinx/connector/delta-lake.md +++ b/docs/src/main/sphinx/connector/delta-lake.md @@ -299,6 +299,8 @@ this table: - `TIMESTAMP(6)` * - `TIMESTAMP` - `TIMESTAMP(3) WITH TIME ZONE` +* - `VARIANT` + - `JSON` * - `ARRAY` - `ARRAY` * - `MAP` diff --git a/docs/src/main/sphinx/connector/hive.md b/docs/src/main/sphinx/connector/hive.md index b4606eba756c..76192dc548e9 100644 --- a/docs/src/main/sphinx/connector/hive.md +++ b/docs/src/main/sphinx/connector/hive.md @@ -417,6 +417,43 @@ limitations and differences: - `GRANT privilege ON SCHEMA schema` is not supported. Schema ownership can be changed with `ALTER SCHEMA schema SET AUTHORIZATION user` +(hive-parquet-encryption)= +## Parquet encryption + +The Hive connector supports reading Parquet files encrypted with Parquet +Modular Encryption (PME). Decryption keys can be provided via environment +variables. Writing encrypted Parquet files is not supported. + +:::{list-table} Parquet encryption properties +:widths: 35, 50, 15 +:header-rows: 1 + +* - Property name + - Description + - Default +* - `pme.environment-key-retriever.enabled` + - Enable the key retriever that reads decryption keys from + environment variables. + - `false` +* - `pme.aad-prefix` + - AAD prefix used when decoding Parquet files. Must match the prefix used + when the files were written, if applicable. + - +* - `pme.check-footer-integrity` + - Validate signature for plaintext footer files. + - `true` +::: + +When `pme.environment-key-retriever.enabled` is set, provide keys with +environment variables: + +- `pme.environment-key-retriever.footer-keys` +- `pme.environment-key-retriever.column-keys` + +Each variable accepts either a single base64-encoded key, or a comma-separated +list of `id:key` pairs (base64-encoded keys) where `id` must match the key +metadata embedded in the Parquet file. + (hive-sql-support)= ## SQL support diff --git a/docs/src/main/sphinx/connector/iceberg.md b/docs/src/main/sphinx/connector/iceberg.md index a95dd2cfa276..c2666f06fc1f 100644 --- a/docs/src/main/sphinx/connector/iceberg.md +++ b/docs/src/main/sphinx/connector/iceberg.md @@ -217,6 +217,15 @@ implementation is used: - Enable [sorted writing](iceberg-sorted-files) to tables with a specified sort order. Equivalent session property is `sorted_writing_enabled`. - `true` +* - `iceberg.sorted-writing.local-staging-path` + - A local directory that Trino can use for staging writes to sorted tables. + The `${USER}` placeholder can be used to use a different + location for each user. When this property is not configured, the target + storage will be used for staging while writing to sorted tables which can + be inefficient when writing to object stores like S3. When + `fs.hadoop.enabled` is not enabled, using this feature requires setup of + [local file system](/object-storage/file-system-local) + - * - `iceberg.allowed-extra-properties` - List of extra properties that are allowed to be set on Iceberg tables. Use `*` to allow all properties. @@ -907,12 +916,39 @@ time is recommended to keep size of a table's data directory under control. ALTER TABLE test_table EXECUTE remove_orphan_files(retention_threshold => '7d'); ``` +```text + metric_name | metric_value +----------------------------+-------------- + processed_manifests_count | 2 + active_files_count | 98 + scanned_files_count | 97 + deleted_files_count | 0 +``` + The value for `retention_threshold` must be higher than or equal to `iceberg.remove-orphan-files.min-retention` in the catalog otherwise the procedure fails with a similar message: `Retention specified (1.00d) is shorter than the minimum retention configured in the system (7.00d)`. The default value for this property is `7d`. +The output of the query has the following metrics: + +:::{list-table} Output +:widths: 40, 60 +:header-rows: 1 + +* - Property name + - Description +* - `processed_manifests_count` + - The count of manifest files read by remove_orphan_files. +* - `active_files_count` + - The count of files belonging to snapshots that have not been expired. +* - `scanned_files_count` + - The count of files scanned from the file system. +* - `deleted_files_count` + - The count of files deleted by remove_orphan_files. +::: + (drop-extended-stats)= ##### drop_extended_stats @@ -2145,7 +2181,8 @@ enabled, metadata caching in coordinator memory is deactivated. Additionally, you can use the following catalog configuration properties: -:::{list-table} Memory metadata caching configuration properties :widths: 25, 75 +:::{list-table} Memory metadata caching configuration properties +:widths: 25, 75 :header-rows: 1 * - Property @@ -2160,4 +2197,4 @@ Additionally, you can use the following catalog configuration properties: Defaults to `200MB`. * - `fs.memory-cache.max-content-length` - The maximum file size that can be cached. Defaults to `15MB`. - ::: + ::: diff --git a/docs/src/main/sphinx/connector/oracle.md b/docs/src/main/sphinx/connector/oracle.md index 034defa9ff1d..88ae659d2ff4 100644 --- a/docs/src/main/sphinx/connector/oracle.md +++ b/docs/src/main/sphinx/connector/oracle.md @@ -18,7 +18,7 @@ like Oracle and Hive, or different Oracle database instances. To connect to Oracle, you need: -- Oracle 19 or higher. +- Oracle 23 or higher. - Network access from the Trino coordinator and workers to Oracle. Port 1521 is the default port. diff --git a/docs/src/main/sphinx/connector/pinot.md b/docs/src/main/sphinx/connector/pinot.md index afc326264384..eaf5e2aeac45 100644 --- a/docs/src/main/sphinx/connector/pinot.md +++ b/docs/src/main/sphinx/connector/pinot.md @@ -36,7 +36,7 @@ This can be the ip or the FQDN, the url scheme (`http://`) is optional. |--------------------------------------------------------|----------| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `pinot.controller-urls` | Yes | A comma separated list of controller hosts. If Pinot is deployed via [Kubernetes](https://kubernetes.io/) this needs to point to the controller service endpoint. The Pinot broker and server must be accessible via DNS as Pinot returns hostnames and not IP addresses. | | `pinot.broker-url` | No | A host and port of broker. If broker URL exposed by Pinot controller API is not accessible, this property can be used to specify the broker endpoint. Enabling this property will disable broker discovery. | -| `pinot.connection-timeout` | No | Pinot connection timeout, default is `15s`. | +| `pinot.connection-timeout` | No | Pinot connection timeout, default is `1m`. | | `pinot.metadata-expiry` | No | Pinot metadata expiration time, default is `2m`. | | `pinot.controller.authentication.type` | No | Pinot authentication method for controller requests. Allowed values are `NONE` and `PASSWORD` - defaults to `NONE` which is no authentication. | | `pinot.controller.authentication.user` | No | Controller username for basic authentication method. | @@ -44,8 +44,8 @@ This can be the ip or the FQDN, the url scheme (`http://`) is optional. | `pinot.broker.authentication.type` | No | Pinot authentication method for broker requests. Allowed values are `NONE` and `PASSWORD` - defaults to `NONE` which is no authentication. | | `pinot.broker.authentication.user` | No | Broker username for basic authentication method. | | `pinot.broker.authentication.password` | No | Broker password for basic authentication method. | -| `pinot.max-rows-per-split-for-segment-queries` | No | Fail query if Pinot server split returns more rows than configured, default to `2,147,483,647`. | -| `pinot.prefer-broker-queries` | No | Pinot query plan prefers to query Pinot broker, default is `true`. | +| `pinot.max-rows-per-split-for-segment-queries` | No | Fail query if Pinot server split returns more rows than configured, default to `2,147,483,646`. | +| `pinot.prefer-broker-queries` | No | Pinot query plan prefers to query Pinot broker, default is `false`. | | `pinot.forbid-segment-queries` | No | Forbid parallel querying and force all querying to happen via the broker, default is `false`. | | `pinot.segments-per-split` | No | The number of segments processed in a split. Setting this higher reduces the number of requests made to Pinot. This is useful for smaller Pinot clusters, default is `1`. | | `pinot.fetch-retry-count` | No | Retry count for retriable Pinot data fetch calls, default is `2`. | @@ -226,7 +226,7 @@ Aggregate function pushdown is enabled by default, but can be disabled with the catalog property `pinot.aggregation-pushdown.enabled` or the catalog session property `aggregation_pushdown_enabled`. -A `count(distint)` pushdown may cause Pinot to run a full table scan with +A `count(distinct)` pushdown may cause Pinot to run a full table scan with significant performance impact. If you encounter this problem, you can disable it with the catalog property `pinot.count-distinct-pushdown.enabled` or the catalog session property `count_distinct_pushdown_enabled`. diff --git a/docs/src/main/sphinx/connector/teradata.md b/docs/src/main/sphinx/connector/teradata.md new file mode 100644 index 000000000000..f6d9f12c9309 --- /dev/null +++ b/docs/src/main/sphinx/connector/teradata.md @@ -0,0 +1,178 @@ +# Teradata connector + +```{raw} html + +``` + +The Teradata connector allows querying and creating tables in an external +[Teradata](https://www.teradata.com/) database. This can be used to join +data between different systems like Teradata and Hive, or between different Teradata instances. + +## Requirements + +To connect to Teradata, you need: + +- Teradata Database +- Network access from the Trino coordinator and workers to Teradata. Port + 1025 is the default port + +## Configuration + +To configure the Teradata connector, create a catalog properties file in +`etc/catalog` named, for example, `teradata.properties`, to mount the Teradata +connector as the `teradata` catalog. Create the file with the following +contents, replacing the connection properties as appropriate for your setup: + +```properties +connector.name=teradata +connection-url=jdbc:teradata://example.teradata.com/CHARSET=UTF8,TMODE=ANSI,LOGMECH=TD2 +connection-user=*** +connection-password=*** +``` + +The `connection-url` defines the connection information and parameters to pass +to the Teradata JDBC driver. The supported parameters for the URL are +available in the +[Teradata JDBC documentation](https://teradata-docs.s3.amazonaws.com/doc/connectivity/jdbc/reference/current/jdbcug_chapter_2.html#BABJIHBJ). +For example, the following `connection-url` configures character encoding, +transaction mode, and authentication. + +```properties +connection-url=jdbc:teradata://example.teradata.com/CHARSET=UTF8,TMODE=ANSI,LOGMECH=TD2 +``` + +The `connection-user` and `connection-password` are typically required and +determine the user credentials for the connection, often a service user. + +### Connection security + +If you have TLS configured with a globally-trusted certificate installed on +your data source, you can enable TLS between your cluster and the data +source by appending parameters to the JDBC connection string set in the +connection-url catalog configuration property. + +For example, to specify SSLMODE: + +```properties +connection-url=jdbc:teradata://example.teradata.com/SSLMODE=REQUIRED +``` + +For more information on TLS configuration options, see the +Teradata [JDBC documentation](https://teradata-docs.s3.amazonaws.com/doc/connectivity/jdbc/reference/current/jdbcug_chapter_2.html#URL_SSLMODE). + +```{include} jdbc-authentication.fragment +``` + +### Multiple Teradata databases + +You can have as many catalogs as you need, so if you have additional Teradata +databases, simply add another properties file to etc/catalog with a different +name, making sure it ends in .properties. +For example, if you name the property file sales.properties, Trino creates a +catalog named sales using the configured connector. + +## Type mapping + +Because Trino and Teradata each support types that the other does not, this +connector {ref}`modifies some types ` when reading data. +Refer to the following sections for type mapping in when reading data from +Teradata to Trino. + +### Teradata type to Trino type mapping + +The connector maps Teradata types to the corresponding Trino types following +this table: + +:::{list-table} Teradata type to Trino type mapping +:widths: 40, 40, 20 +:header-rows: 1 + +* - Teradata type + - Trino type + - Notes +* - `TINYINT` + - `TINYINT` + - +* - `SMALLINT` + - `SMALLINT` + - +* - `INTEGER` + - `INTEGER` + - +* - `BIGINT` + - `BIGINT` + - +* - `REAL` + - `DOUBLE` + - +* - `DOUBLE` + - `DOUBLE` + - +* - `FLOAT` + - `DOUBLE` + - +* - `NUMBER(p, s)` + - `DECIMAL(p, s)` + - +* - `NUMERIC(p, s)` + - `DECIMAL(p, s)` + - +* - `DECIMAL(p, s)` + - `DECIMAL(p, s)` + - +* - `CHAR(n)` + - `CHAR(n)` + - +* - `CHARACTER(n)` + - `CHAR(n)` + - +* - `VARCHAR(n)` + - `VARCHAR(n)` + - +* - `DATE` + - `DATE` + - +::: + +No other types are supported. + +```{include} jdbc-type-mapping.fragment +``` + +## Querying Teradata + +The Teradata connector provides a schema for every Teradata database. You can +see the available Teradata databases by running SHOW SCHEMAS: + +``` +SHOW SCHEMAS FROM teradata; +``` + +If you have a Teradata database named sales, you can view the tables in this +database by running SHOW TABLES: + +``` +SHOW TABLES FROM teradata.sales; +``` + +You can see a list of the columns in the orders table in the sales database +using either of the following: + +``` +DESCRIBE teradata.sales.orders; +SHOW COLUMNS FROM teradata.sales.orders; +``` + +Finally, you can access the orders table in the sales database: + +``` +SELECT * FROM teradata.sales.orders; +``` + +## SQL support + +The connector provides read access access to data and metadata in +a Teradata database. The connector supports the {ref}`globally available +` and {ref}`read operation ` +statements. + diff --git a/docs/src/main/sphinx/develop/example-http.md b/docs/src/main/sphinx/develop/example-http.md index e6ebc9817f5d..76688f7da29d 100644 --- a/docs/src/main/sphinx/develop/example-http.md +++ b/docs/src/main/sphinx/develop/example-http.md @@ -14,7 +14,7 @@ image](/installation/containers). Follow the [plugin installation instructions](plugins-installation) and optionally use the [trino-packages project](https://github.com/trinodb/trino-packages) or manually download the -plugin archive {maven_download}`example-http`. +plugin archive {download_gh}`example-http`. ## Code diff --git a/docs/src/main/sphinx/ext/download.py b/docs/src/main/sphinx/ext/download.py index 058f2c0e9558..2dce7a7c4ef2 100644 --- a/docs/src/main/sphinx/ext/download.py +++ b/docs/src/main/sphinx/ext/download.py @@ -84,8 +84,15 @@ def filename(artifact, version, extension): extension = '.' + extension if extension else '' return artifact + '-' + version + extension +# Download from Maven Central +def download_mc_url(artifact, version, extension): + base = 'https://repo1.maven.org/maven2/io/trino' + file = filename(artifact, version, extension) + return '%s/%s/%s/%s' % (base, artifact, version, file) + -def download_url(artifact, version, extension): +# Download from GitHub Releases +def download_gh_url(artifact, version, extension): base = 'https://github.com/trinodb/trino/releases/download' file = filename(artifact, version, extension) return '%s/%s/%s' % (base, version, file) @@ -93,7 +100,7 @@ def download_url(artifact, version, extension): def setup(app): # noinspection PyDefaultArgument,PyUnusedLocal - def download_link_role(role, rawtext, text, lineno, inliner, options={}, content=[]): + def download_gh_link_role(role, rawtext, text, lineno, inliner, options={}, content=[]): version = app.config.release if not text in ARTIFACTS: @@ -103,12 +110,30 @@ def download_link_role(role, rawtext, text, lineno, inliner, options={}, content artifact, extension = ARTIFACTS[text] title = filename(artifact, version, extension) - uri = download_url(artifact, version, extension) + uri = download_gh_url(artifact, version, extension) node = nodes.reference(title, title, internal=False, refuri=uri) return [node], [] - app.add_role('maven_download', download_link_role) + + def download_mc_link_role(role, rawtext, text, lineno, inliner, options={}, content=[]): + version = app.config.release + + if not text in ARTIFACTS: + inliner.reporter.error('Unsupported download type: ' + text, line=lineno) + return [], [] + + artifact, extension = ARTIFACTS[text] + + title = filename(artifact, version, extension) + uri = download_mc_url(artifact, version, extension) + + node = nodes.reference(title, title, internal=False, refuri=uri) + + return [node], [] + + app.add_role('download_gh', download_gh_link_role) + app.add_role('download_mc', download_mc_link_role) return { 'parallel_read_safe': True, diff --git a/docs/src/main/sphinx/installation/deployment.md b/docs/src/main/sphinx/installation/deployment.md index 356a2b32fe3b..7c95182e06d2 100644 --- a/docs/src/main/sphinx/installation/deployment.md +++ b/docs/src/main/sphinx/installation/deployment.md @@ -42,12 +42,12 @@ Docker image](https://hub.docker.com/r/trinodb/trino). ## Installing Trino -Download the Trino server tarball, {maven_download}`server`, and unpack it. The +Download the Trino server tarball, {download_gh}`server`, and unpack it. The tarball contains a single top-level directory, `trino-server-|trino_version|`, which we call the *installation* directory. The default tarball contains all plugins and must be configured for use. The -minimal `server-core` tarball, {maven_download}`server-core`, contains a minimal +minimal `server-core` tarball, {download_gh}`server-core`, contains a minimal set of essential plugins, and it is therefore mostly suitable as a base for custom tarball creation. diff --git a/docs/src/main/sphinx/installation/plugins.md b/docs/src/main/sphinx/installation/plugins.md index b366731aa13b..6537ca6f41cf 100644 --- a/docs/src/main/sphinx/installation/plugins.md +++ b/docs/src/main/sphinx/installation/plugins.md @@ -147,221 +147,221 @@ with the listed coordinates. * - ai-functions - [](/functions/ai) - [io.trino:trino-ai-functions](https://central.sonatype.com/search?q=io.trino%3Atrino-ai-functions) - - {maven_download}`ai-functions` + - {download_gh}`ai-functions` * - bigquery - [](/connector/bigquery) - [io.trino:trino-bigquery](https://central.sonatype.com/search?q=io.trino%3Atrino-bigquery) - - {maven_download}`bigquery` + - {download_gh}`bigquery` * - blackhole - [](/connector/blackhole) - [io.trino:trino-blackhole](https://central.sonatype.com/search?q=io.trino%3Atrino-blackhole) - - {maven_download}`blackhole` + - {download_gh}`blackhole` * - cassandra - [](/connector/cassandra) - [io.trino:trino-cassandra](https://central.sonatype.com/search?q=io.trino%3Atrino-cassandra) - - {maven_download}`cassandra` + - {download_gh}`cassandra` * - clickhouse - [](/connector/clickhouse) - [io.trino:trino-clickhouse](https://central.sonatype.com/search?q=io.trino%3Atrino-clickhouse) - - {maven_download}`clickhouse` + - {download_gh}`clickhouse` * - delta-lake - [](/connector/delta-lake) - [io.trino:trino-delta-lake](https://central.sonatype.com/search?q=io.trino%3Atrino-delta-lake) - - {maven_download}`delta-lake` + - {download_gh}`delta-lake` * - druid - [](/connector/druid) - [io.trino:trino-druid](https://central.sonatype.com/search?q=io.trino%3Atrino-druid) - - {maven_download}`druid` + - {download_gh}`druid` * - duckdb - [](/connector/duckdb) - [io.trino:trino-duckdb](https://central.sonatype.com/search?q=io.trino%3Atrino-duckdb) - - {maven_download}`duckdb` + - {download_gh}`duckdb` * - elasticsearch - [](/connector/elasticsearch) - [io.trino:trino-elasticsearch](https://central.sonatype.com/search?q=io.trino%3Atrino-elasticsearch) - - {maven_download}`elasticsearch` + - {download_gh}`elasticsearch` * - example-http - [](/develop/example-http) - [io.trino:trino-example-http](https://central.sonatype.com/search?q=io.trino%3Atrino-example-http) - - {maven_download}`example-http` + - {download_gh}`example-http` * - exasol - [](/connector/exasol) - [io.trino:trino-exasol](https://central.sonatype.com/search?q=io.trino%3Atrino-exasol) - - {maven_download}`exasol` + - {download_gh}`exasol` * - exchange-filesystem - [](/admin/fault-tolerant-execution) exchange file system - [io.trino:trino-exchange-filesystem](https://central.sonatype.com/search?q=io.trino%3Atrino-exchange-filesystem) - - {maven_download}`exchange-filesystem` + - {download_gh}`exchange-filesystem` * - exchange-hdfs - [](/admin/fault-tolerant-execution) exchange file system for HDFS - [io.trino:trino-exchange-hdfs](https://central.sonatype.com/search?q=io.trino%3Atrino-exchange-hdfs) - - {maven_download}`exchange-hdfs` + - {download_gh}`exchange-hdfs` * - faker - [](/connector/faker) - [io.trino:trino-faker](https://central.sonatype.com/search?q=io.trino%3Atrino-faker) - - {maven_download}`faker` + - {download_gh}`faker` * - functions-python - [](/udf/python) - [io.trino:trino-functions-python](https://central.sonatype.com/search?q=io.trino%3Atrino-functions-python) - - {maven_download}`functions-python` + - {download_gh}`functions-python` * - geospatial - [](/functions/geospatial) - [io.trino:trino-geospatial](https://central.sonatype.com/search?q=io.trino%3Atrino-geospatial) - - {maven_download}`geospatial` + - {download_gh}`geospatial` * - google-sheets - [](/connector/googlesheets) - [io.trino:trino-google-sheets](https://central.sonatype.com/search?q=io.trino%3Atrino-google-sheets) - - {maven_download}`google-sheets` + - {download_gh}`google-sheets` * - hive - [](/connector/hive) - [io.trino:trino-hive](https://central.sonatype.com/search?q=io.trino%3Atrino-hive) - - {maven_download}`hive` + - {download_gh}`hive` * - http-event-listener - [](/admin/event-listeners-http) - [io.trino:trino-http-event-listener](https://central.sonatype.com/search?q=io.trino%3Atrino-http-event-listener) - - {maven_download}`http-event-listener` + - {download_gh}`http-event-listener` * - http-server-event-listener - HTTP server event listener - [io.trino:trino-http-server-event-listener](https://central.sonatype.com/search?q=io.trino%3Atrino-http-server-event-listener) - - {maven_download}`http-server-event-listener` + - {download_gh}`http-server-event-listener` * - hudi - [](/connector/hudi) - [io.trino:trino-hudi](https://central.sonatype.com/search?q=io.trino%3Atrino-hudi) - - {maven_download}`hudi` + - {download_gh}`hudi` * - iceberg - [](/connector/iceberg) - [io.trino:trino-iceberg](https://central.sonatype.com/search?q=io.trino%3Atrino-iceberg) - - {maven_download}`iceberg` + - {download_gh}`iceberg` * - ignite - [](/connector/ignite) - [io.trino:trino-ignite](https://central.sonatype.com/search?q=io.trino%3Atrino-ignite) - - {maven_download}`ignite` + - {download_gh}`ignite` * - jmx - [](/connector/jmx) - [io.trino:trino-jmx](https://central.sonatype.com/search?q=io.trino%3Atrino-jmx) - - {maven_download}`jmx` + - {download_gh}`jmx` * - kafka - [](/connector/kafka) - [io.trino:trino-kafka](https://central.sonatype.com/search?q=io.trino%3Atrino-kafka) - - {maven_download}`kafka` + - {download_gh}`kafka` * - kafka-event-listener - [](/admin/event-listeners-kafka) - [io.trino:trino-kafka-event-listener](https://central.sonatype.com/search?q=io.trino%3Atrino-kafka-event-listener) - - {maven_download}`kafka-event-listener` + - {download_gh}`kafka-event-listener` * - loki - [](/connector/loki) - [io.trino:trino-loki](https://central.sonatype.com/search?q=io.trino%3Atrino-loki) - - {maven_download}`loki` + - {download_gh}`loki` * - mariadb - [](/connector/mariadb) - [io.trino:trino-mariadb](https://central.sonatype.com/search?q=io.trino%3Atrino-mariadb) - - {maven_download}`mariadb` + - {download_gh}`mariadb` * - memory - [](/connector/memory) - [io.trino:trino-memory](https://central.sonatype.com/search?q=io.trino%3Atrino-memory) - - {maven_download}`memory` + - {download_gh}`memory` * - ml - [](/functions/ml) - [io.trino:trino-ml](https://central.sonatype.com/search?q=io.trino%3Atrino-ml) - - {maven_download}`ml` + - {download_gh}`ml` * - mongodb - [](/connector/mongodb) - [io.trino:trino-mongodb](https://central.sonatype.com/search?q=io.trino%3Atrino-mongodb) - - {maven_download}`mongodb` + - {download_gh}`mongodb` * - mysql - [](/connector/mysql) - [io.trino:trino-mysql](https://central.sonatype.com/search?q=io.trino%3Atrino-mysql) - - {maven_download}`mysql` + - {download_gh}`mysql` * - mysql-event-listener - [](/admin/event-listeners-mysql) - [io.trino:trino-mysql-event-listener](https://central.sonatype.com/search?q=io.trino%3Atrino-mysql-event-listener) - - {maven_download}`mysql-event-listener` + - {download_gh}`mysql-event-listener` * - opa - [](/security/opa-access-control) - [io.trino:trino-opa](https://central.sonatype.com/search?q=io.trino%3Atrino-opa) - - {maven_download}`opa` + - {download_gh}`opa` * - openlineage - [](/admin/event-listeners-openlineage) - [io.trino:trino-openlineage](https://central.sonatype.com/search?q=io.trino%3Atrino-openlineage) - - {maven_download}`openlineage` + - {download_gh}`openlineage` * - opensearch - [](/connector/opensearch) - [io.trino:trino-opensearch](https://central.sonatype.com/search?q=io.trino%3Atrino-opensearch) - - {maven_download}`opensearch` + - {download_gh}`opensearch` * - oracle - [](/connector/oracle) - [io.trino:trino-oracle](https://central.sonatype.com/search?q=io.trino%3Atrino-oracle) - - {maven_download}`oracle` + - {download_gh}`oracle` * - password-authenticators - Password authentication - [io.trino:trino-password-authenticators](https://central.sonatype.com/search?q=io.trino%3Atrino-password-authenticators) - - {maven_download}`password-authenticators` + - {download_gh}`password-authenticators` * - pinot - [](/connector/pinot) - [io.trino:trino-pinot](https://central.sonatype.com/search?q=io.trino%3Atrino-pinot) - - {maven_download}`pinot` + - {download_gh}`pinot` * - postgresql - [](/connector/postgresql) - [io.trino:trino-postgresql](https://central.sonatype.com/search?q=io.trino%3Atrino-postgresql) - - {maven_download}`postgresql` + - {download_gh}`postgresql` * - prometheus - [](/connector/prometheus) - [io.trino:trino-prometheus](https://central.sonatype.com/search?q=io.trino%3Atrino-prometheus) - - {maven_download}`prometheus` + - {download_gh}`prometheus` * - ranger - [](/security/ranger-access-control) - [io.trino:trino-ranger](https://central.sonatype.com/search?q=io.trino%3Atrino-ranger) - - {maven_download}`ranger` + - {download_gh}`ranger` * - redis - [](/connector/redis) - [io.trino:trino-redis](https://central.sonatype.com/search?q=io.trino%3Atrino-redis) - - {maven_download}`redis` + - {download_gh}`redis` * - redshift - [](/connector/redshift) - [io.trino:trino-redshift](https://central.sonatype.com/search?q=io.trino%3Atrino-redshift) - - {maven_download}`redshift` + - {download_gh}`redshift` * - resource-group-managers - [](/admin/resource-groups) - [io.trino:trino-resource-group-managers](https://central.sonatype.com/search?q=io.trino%3Atrino-resource-group-managers) - - {maven_download}`resource-group-managers` + - {download_gh}`resource-group-managers` * - session-property-managers - [](/admin/session-property-managers) - [io.trino:trino-session-property-managers](https://central.sonatype.com/search?q=io.trino%3Atrino-session-property-managers) - - {maven_download}`session-property-managers` + - {download_gh}`session-property-managers` * - singlestore - [](/connector/singlestore) - [io.trino:trino-singlestore](https://central.sonatype.com/search?q=io.trino%3Atrino-singlestore) - - {maven_download}`singlestore` + - {download_gh}`singlestore` * - snowflake - [](/connector/snowflake) - [io.trino:trino-snowflake](https://central.sonatype.com/search?q=io.trino%3Atrino-snowflake) - - {maven_download}`snowflake` + - {download_gh}`snowflake` * - spooling-filesystem - Server side support for [](protocol-spooling) - [io.trino:trino-spooling-filesystem](https://central.sonatype.com/search?q=io.trino%3Atrino-spooling-filesystem) - - {maven_download}`spooling-filesystem` + - {download_gh}`spooling-filesystem` * - sqlserver - [](/connector/sqlserver) - [io.trino:trino-sqlserver](https://central.sonatype.com/search?q=io.trino%3Atrino-sqlserver) - - {maven_download}`sqlserver` + - {download_gh}`sqlserver` * - teradata-functions - [](/functions/teradata) - [io.trino:trino-teradata-functions](https://central.sonatype.com/search?q=io.trino%3Atrino-teradata-functions) - - {maven_download}`teradata-functions` + - {download_gh}`teradata-functions` * - thrift - [](/connector/thrift) - [io.trino:trino-thrift](https://central.sonatype.com/search?q=io.trino%3Atrino-thrift) - - {maven_download}`thrift` + - {download_gh}`thrift` * - tpcds - [](/connector/tpcds) - [io.trino:trino-tpcds](https://central.sonatype.com/search?q=io.trino%3Atrino-tpcds) - - {maven_download}`tpcds` + - {download_gh}`tpcds` * - tpch - [](/connector/tpch) - [io.trino:trino-tpch](https://central.sonatype.com/search?q=io.trino%3Atrino-tpch) - - {maven_download}`tpch` + - {download_gh}`tpch` * - vertica - [](/connector/vertica) - [io.trino:trino-vertica](https://central.sonatype.com/search?q=io.trino%3Atrino-vertica) - - {maven_download}`vertica` + - {download_gh}`vertica` ::: \ No newline at end of file diff --git a/docs/src/main/sphinx/object-storage/file-formats.md b/docs/src/main/sphinx/object-storage/file-formats.md index 046d821984a8..c192e6ad5fe9 100644 --- a/docs/src/main/sphinx/object-storage/file-formats.md +++ b/docs/src/main/sphinx/object-storage/file-formats.md @@ -31,6 +31,14 @@ with ORC files performed by supported object storage connectors: [](file-compression) is automatically performed and some details can be configured. +## ORC support limitations + +[Trino ignores Calendar entry in ORC file metadata.](https://github.com/trinodb/trino/issues/26865) +As a result Trino always treats dates and timestamps as values written using +proleptic Gregorian calendar. This causes incorrect values read when reading +date/time values before Oct 15, 1582 that were written using hybrid +Julian-Gregorian calendar. + (parquet-format-configuration)= ## Parquet format configuration properties diff --git a/docs/src/main/sphinx/object-storage/file-system-gcs.md b/docs/src/main/sphinx/object-storage/file-system-gcs.md index 3bde8515b782..49df7ca4a86f 100644 --- a/docs/src/main/sphinx/object-storage/file-system-gcs.md +++ b/docs/src/main/sphinx/object-storage/file-system-gcs.md @@ -70,7 +70,15 @@ Cloud Storage: - Description * - `gcs.use-access-token` - Flag to set usage of a client-provided OAuth 2.0 token to access Google - Cloud Storage. Defaults to `false`. + Cloud Storage. Defaults to `false`, deprecated to use `gcs.auth-type` instead. +* - `gcs.auth-type` + - Authentication type to use for Google Cloud Storage access. Default to `SERVICE_ACCOUNT`. + Supported values are: + * `SERVICE_ACCOUNT`: loads credentials from the environment. Either `gcs.json-key` or + `gcs.json-key-file-path` can be set in addition to override the default + credentials provider. + * `ACCESS_TOKEN`: usage of client-provided OAuth 2.0 token to access Google + Cloud Storage. * - `gcs.json-key` - Your Google Cloud service account key in JSON format. Not to be set together with `gcs.json-key-file-path`. @@ -103,7 +111,7 @@ Cloud Storage, make the following edits to your catalog configuration: - Native property - Notes * - `hive.gcs.use-access-token` - - `gcs.use-access-token` + - `gcs.auth-type` - * - `hive.gcs.json-key-file-path` - `gcs.json-key-file-path` diff --git a/docs/src/main/sphinx/release.md b/docs/src/main/sphinx/release.md index 76790a997260..ec9e4f6e9fb2 100644 --- a/docs/src/main/sphinx/release.md +++ b/docs/src/main/sphinx/release.md @@ -6,6 +6,7 @@ ```{toctree} :maxdepth: 1 +release/release-478 release/release-477 release/release-476 release/release-475 diff --git a/docs/src/main/sphinx/release/release-477.md b/docs/src/main/sphinx/release/release-477.md index d41af6eb991a..3dbea2885536 100644 --- a/docs/src/main/sphinx/release/release-477.md +++ b/docs/src/main/sphinx/release/release-477.md @@ -29,7 +29,7 @@ * Do not include catalogs that failed to load in `system.metadata.catalogs`. ({issue}`26493`) * Simplify node discovery configuration for Kubernetes-like environments that provide DNS names for all workers when the `discovery.type` config property is set - to `dns`. ({issue}`26119`, {issue}`26119`) + to `dns`. ({issue}`26119`) * Improve memory usage for certain queries involving {func}`row_number`, {func}`rank`, {func}`dense_rank`, and `ORDER BY ... LIMIT`. ({issue}`25946`) * Improve memory usage for queries involving `GROUP BY`. ({issue}`25879`) @@ -218,7 +218,7 @@ ## SPI * Remove `ConnectorSession` from `Type.getObjectValue`. ({issue}`25945`) -* Remove unused `NodeManager` `getEnvironment` method. ({issue}`26096`) +* Remove unused `NodeManager.getEnvironment` method. ({issue}`26096`) * Remove `@Experimental` annotation. ({issue}`26200`) * Remove deprecated `ConnectorPageSource.getNextPage` method. ({issue}`26222`) * Remove support for `EventListener#splitCompleted`. ({issue}`26436`) diff --git a/docs/src/main/sphinx/release/release-478.md b/docs/src/main/sphinx/release/release-478.md new file mode 100644 index 000000000000..a0bac4237614 --- /dev/null +++ b/docs/src/main/sphinx/release/release-478.md @@ -0,0 +1,74 @@ +# Release 478 (29 Oct 2025) + +## General + +* Include lineage information for columns used in `UNNEST` expressions. ({issue}`16946`) +* Add support for limiting which retry policies a user can select. This can be configured using + the `retry-policy.allowed` option. ({issue}`26628`) +* Add support for loading plugins from multiple directories. ({issue}`26855`) +* Allow dropping catalogs that failed to load correctly. ({issue}`26918`) +* Improve performance of queries with an `ORDER BY` clause using `varchar` or `varbinary` types. ({issue}`26725`) +* Improve performance of `MERGE` statements involving a `NOT MATCHED` case. ({issue}`26759`) +* Improve performance of queries involving `JOIN` when the join spills to disk. ({issue}`26076`) +* Fix potential incorrect results when query uses `row` type. ({issue}`26806`) +* Include catalogs that failed to load in the `metadata.catalogs` table. ({issue}`26918`) +* Fix `EXPLAIN ANALYZE` planning so that it executes with the same plan as would be used to execute the query + being analyzed. ({issue}`26938`) +* Fix incorrect results when using logical navigation function `FIRST` in row pattern recognition. ({issue}`26981`) + +## Security + +* Propagate `queryId` to the [Open Policy Agent](/security/opa-access-control) + authorizer. ({issue}`26851`) + +## Docker image + +* Run Trino on JDK 25.0.0 (build 36). ({issue}`26693`) + +## Delta Lake connector + +* Fix failure when reading a `map(..., json)` column when the map item value is `NULL`. ({issue}`26700`) +* Deprecate the `gcs.use-access-token` configuration property. Use `gcs.auth-type` instead. ({issue}`26681`) + +## Google Sheets connector + +* Fix potential query failure when the `gsheets.delegated-user-email` configuration property + is used. ({issue}`26501`) + +## Hive connector + +* Add support for reading encrypted Parquet files. ({issue}`24517`, {issue}`9383`) +* Deprecate the `gcs.use-access-token` configuration property. Use `gcs.auth-type` instead. ({issue}`26681`) +* Improve performance of queries using complex predicates on `$path` column. ({issue}`27000`) +* Fix writing ORC files to ensure that dates and timestamps before `1582-10-15` are read correctly by Apache Hive. ({issue}`26507`) +* Fix `flush_metadata_cache` procedure failure when metastore impersonation is enabled. ({issue}`27059`) + +## Hudi connector + +* Deprecate the `gcs.use-access-token` configuration property. Use `gcs.auth-type` instead. ({issue}`26681`) + +## Iceberg connector + +* Improve performance when writing sorted tables and `iceberg.sorted-writing.local-staging-path` + is set. ({issue}`24376`) +* Improve performance of `ALTER TABLE EXECUTE OPTIMIZE` on tables with bucket transform partitioning. ({issue}`27104`) +* Return execution metrics while running the `remove_orphan_files` command. ({issue}`26661`) +* Deprecate the `gcs.use-access-token` configuration property. Use `gcs.auth-type` instead. ({issue}`26681`) +* Collect distinct values count on all columns when replacing tables. ({issue}`26983`) +* Fix failure due to column count mismatch when executing the `add_files_from_table` + procedure. ({issue}`26774`) +* Fix failure when executing `optimize_manifests` on tables without a snapshot. ({issue}`26970`) +* Fix incorrect results when reading Avro files migrated from Hive. ({issue}`26863`) +* Fix failure when executing `SHOW CREATE SCHEMA` on a schema with unsupported properties + with REST, Glue or Nessie catalog. ({issue}`24744`) +* Fix failure when running `EXPLAIN` or `EXPLAIN ANALYZE` on `OPTIMIZE` command. ({issue}`26598`) + +## Kafka connector + +* Fix failure when filtering partitions by timestamp offset. ({issue}`26787`) + +## SPI + +* Remove default implementation from `Connector.shutdown()`. ({issue}`26718`) +* Remove the deprecated `ConnectorSplit.getSplitInfo` method. ({issue}`27063`) +* Deprecate `io.trino.spi.type.Type#appendTo` method. ({issue}`26922`) diff --git a/docs/src/main/sphinx/security/group-mapping.md b/docs/src/main/sphinx/security/group-mapping.md index e2e52b0cfcc0..601c37a354f5 100644 --- a/docs/src/main/sphinx/security/group-mapping.md +++ b/docs/src/main/sphinx/security/group-mapping.md @@ -1,6 +1,6 @@ # Group mapping -Group providers in Trino to map usernames onto groups for easier access control +Group providers in Trino map usernames onto groups for easier access control and resource group management. Configure a group provider by creating an `etc/group-provider.properties` file @@ -9,8 +9,8 @@ on the coordinator: ```properties group-provider.name=file ``` -_The configuration of the chosen group provider should be appended to this -file_ +The value for `group-provider.name` must be either `file` or `ldap` and the +configuration of the chosen group provider must be included in the same file. :::{list-table} Group provider configuration :widths: 40, 60 @@ -24,7 +24,7 @@ file_ Supported values are: * `file`: [See configuration](file-group-provider) - * `ldap` + * `ldap`: [See configuration](ldap-group-provider) * - `group-provider.group-case` - Optional transformation of the case of the group name. Supported values are: @@ -36,14 +36,21 @@ file_ Defaults to `keep`. ::: +## Integration with access control + +Groups resolved by the group provider are passed to Trino’s system access +control engine. Access control rules can reference these group names to grant +or restrict permissions. + (file-group-provider)= ## File group provider -Group file resolves group membership using a file on the coordinator. +The file group provider resolves group memberships with the configuration in +the group-provider.properties file on the coordinator. -### Group file configuration +### Configuration -Enable group file by creating an `etc/group-provider.properties` +Enable the file group provider by creating an `etc/group-provider.properties` file on the coordinator: ```properties @@ -59,14 +66,12 @@ The following configuration properties are available: * - Property name - Description - * - `file.group-file` - - Path of the group file + - Path of the group file. * - `file.refresh-period` - [Duration](prop-type-duration) between refreshing the group mapping - configuration from the file. - Defaults to `5s`. - ::: + configuration from the file. Defaults to `5s`. +::: ### Group file format @@ -77,3 +82,133 @@ separated by a colon. Users are separated by a comma. group_name:user_1,user_2,user_3 ``` +(ldap-group-provider)= +## LDAP group provider + +The LDAP group provider resolves user group memberships from configuration +retrieved from an LDAP server. This allows access rules to be defined based on +LDAP groups instead of individual users. + +### Configuration + +Enable LDAP group provider by creating an `etc/group-provider.properties` file +on the coordinator and add further configuration for the LDAP server +connections and other information as detailed in the following sections. + +```properties +group-provider.name=ldap +``` + +:::{list-table} Generic LDAP properties +:widths: 40, 60 +:header-rows: 1 +* - Property name + - Description +* - `ldap.url` + - LDAP server URI. For example, `ldap://host:389` or `ldaps://host:636`. +* - `ldap.allow-insecure` + - Allow insecure connection to the LDAP server. Defaults to `false`. +* - `ldap.ssl.keystore.path` + - Path to the PEM or JKS key store. +* - `ldap.ssl.keystore.password` + - Password for the key store. +* - `ldap.ssl.truststore.path` + - Path to the PEM or JKS trust store. +* - `ldap.ssl.truststore.password` + - Password for the trust store. +* - `ldap.ignore-referrals` + - Referrals allow finding entries across multiple LDAP servers. Ignore them + to only search within one LDAP server. Defaults to `false`. +* - `ldap.timeout.connect` + - Timeout [duration](prop-type-duration) for establishing a connection. + Defaults to `1m`. +* - `ldap.timeout.read` + - Timeout [duration](prop-type-duration) for reading data from LDAP. + Defaults to `1m`. +* - `ldap.admin-user` + - Bind distinguished name for admin user. For example, + `CN=UserName,OU=City,OU=State,DC=domain,DC=domain_root` +* - `ldap.admin-password` + - Bind password used for the admin user. +* - `ldap.user-base-dn` + - Base distinguished name for users. For example, `dc=example,dc=com`. +* - `ldap.user-search-filter` + - LDAP filter to find user entries; `{0}` is replaced with the Trino username. + For example, `(cn={0})` +* - `ldap.group-name-attribute` + - Attribute to extract group name from group entry. For example, `cn`. +* - `ldap.use-group-filter` + - Whether to use search-based group resolution. Defaults to `true`. + When `false`, Trino uses the attribute-based method. +::: + +Group resolution behavior is controlled by the `ldap.use-group-filter` property. +With search-based group resolution, Trino searches for group entries that +include the user DN. This requires the following properties: + +:::{list-table} Search-based group resolution +:widths: 40, 60 +:header-rows: 1 +* - Property name + - Description +* - `ldap.group-base-dn` + - Base distinguished name for groups. For example, `dc=example,dc=com`. +* - `ldap.group-search-filter` + - Search filter for group documents. For example, `(cn=trino_*)`. +* - `ldap.group-search-member-attribute` + - Attribute from group documents used for filtering by member. For example, + `cn`. +::: + +In case of attribute-based group resolution, Trino reads the group list +directly from a user attribute. This requires the following property: + +:::{list-table} Attribute-based (single query) group resolution +:widths: 40, 60 +:header-rows: 1 + +* - Property name + - Description +* - `ldap.user-member-of-attribute` + - Group membership attribute in user documents. For example, `memberOf`. +::: + +### Example configurations + +The following configuration is an example for an OpenLDAP (search-based) +group provider: + +```properties +group-provider.name=ldap +group-provider.group-case=lower + +ldap.url=ldap://ldap.example.com:389 +ldap.admin-user=cn=admin,dc=example,dc=com +ldap.admin-password=your_password +ldap.group-name-attribute=cn +ldap.user-base-dn=ou=users,dc=example,dc=com +ldap.user-search-filter=(uid={0}) +ldap.use-group-filter=true + +ldap.group-base-dn=ou=groups,dc=example,dc=com +ldap.group-search-filter=(cn=trino_*) +ldap.group-search-member-attribute=member +``` + +The following configuration is an example for an Active Directory +(single query, attribute-based) group provider: + +```properties +group-provider.name=ldap +group-provider.group-case=lower + +ldap.url=ldaps://ad.example.com:636 +ldap.admin-user=cn=admin,dc=example,dc=com +ldap.admin-password=your_password +ldap.group-name-attribute=cn +ldap.user-base-dn=ou=users,dc=example,dc=com +ldap.user-search-filter=(sAMAccountName={0}) +ldap.use-group-filter=false + +ldap.user-member-of-attribute=memberOf +``` diff --git a/docs/src/main/sphinx/security/opa-access-control.md b/docs/src/main/sphinx/security/opa-access-control.md index 6a16bd9b6ba0..5df693dca5b3 100644 --- a/docs/src/main/sphinx/security/opa-access-control.md +++ b/docs/src/main/sphinx/security/opa-access-control.md @@ -130,6 +130,7 @@ The `context` object contains all other contextual information about the query: following two fields: - `user`: username - `groups`: list of groups this user belongs to +- `queryId`: Query id - `softwareStack`: Information about the software stack issuing the request to OPA. The following information is included: - `trinoVersion`: Version of Trino used @@ -158,6 +159,7 @@ Accessing a table results in a query similar to the following example: "user": "foo", "groups": ["some-group"] }, + "queryId": "20250718_081710_03427_trino", "softwareStack": { "trinoVersion": "434" } @@ -190,6 +192,7 @@ The `targetResource` is used in cases where a new resource, distinct from the on "user": "foo", "groups": ["some-group"] }, + "queryId": "20250718_081710_03427_trino", "softwareStack": { "trinoVersion": "434" } @@ -373,6 +376,7 @@ A batch column masking request is similar to the following example: "user": "foo", "groups": ["some-group"] }, + "queryId": "20250718_081710_03427_trino", "softwareStack": { "trinoVersion": "434" } diff --git a/docs/src/main/sphinx/security/ranger-access-control.md b/docs/src/main/sphinx/security/ranger-access-control.md index a1e5e4cdbb43..897680f2e3a0 100644 --- a/docs/src/main/sphinx/security/ranger-access-control.md +++ b/docs/src/main/sphinx/security/ranger-access-control.md @@ -193,3 +193,7 @@ The following table lists the configuration properties for the Ranger access con execute any query. * To allow this, create a policy in Apache Ranger for a `trinouser` resource with value `{USER}` and with the `impersonate` permission for user `{USER}`. +* External clients can use the `system.runtime.kill_query()` procedure to + kill running queries. Add a policy with Schema `system`, Database + `runtime` and Procedure `kill_query` with `execute` permission for user + `{USER}`. diff --git a/docs/src/main/sphinx/sql.md b/docs/src/main/sphinx/sql.md index cc09ea93e74a..03c20b99d213 100644 --- a/docs/src/main/sphinx/sql.md +++ b/docs/src/main/sphinx/sql.md @@ -11,6 +11,7 @@ Refer to the following sections for further details: ```{toctree} :maxdepth: 1 +sql/alter-branch sql/alter-materialized-view sql/alter-schema sql/alter-table @@ -19,6 +20,7 @@ sql/analyze sql/call sql/comment sql/commit +sql/create-branch sql/create-catalog sql/create-function sql/create-materialized-view @@ -33,6 +35,7 @@ sql/deny sql/describe sql/describe-input sql/describe-output +sql/drop-branch sql/drop-catalog sql/drop-function sql/drop-materialized-view @@ -62,6 +65,7 @@ sql/set-role sql/set-session sql/set-session-authorization sql/set-time-zone +sql/show-branches sql/show-catalogs sql/show-columns sql/show-create-function diff --git a/docs/src/main/sphinx/sql/alter-branch.md b/docs/src/main/sphinx/sql/alter-branch.md new file mode 100644 index 000000000000..874f2eb6a94e --- /dev/null +++ b/docs/src/main/sphinx/sql/alter-branch.md @@ -0,0 +1,26 @@ +# ALTER BRANCH + +## Synopsis + +```text +ALTER BRANCH source_branch IN TABLE table_name FAST FORWARD TO target_branch +``` + +## Description + +Fast-forward the current snapshot of one branch to the latest snapshot of +another. + +## Examples + +Fast-forward the `main` branch to the head of `audit` branch in the `orders` +table: + +```sql +ALTER BRANCH main IN TABLE orders FAST FORWARD TO audit +``` + +## See also + +- {doc}`create-branch` +- {doc}`drop-branch` diff --git a/docs/src/main/sphinx/sql/create-branch.md b/docs/src/main/sphinx/sql/create-branch.md new file mode 100644 index 000000000000..bfed6d675d31 --- /dev/null +++ b/docs/src/main/sphinx/sql/create-branch.md @@ -0,0 +1,46 @@ +# CREATE BRANCH + +## Synopsis + +```text +CREATE [ OR REPLACE ] BRANCH [ IF NOT EXISTS ] branch_name +[ WITH ( property_name = expression [, ...] ) ] +IN TABLE table_name +[ FROM source_branch ] +``` + +## Description + +Create a branch. + +The optional `OR REPLACE` clause causes an existing branch with the +specified name to be replaced with the new branch definition. Support +for branch replacement varies across connectors. Refer to the +connector documentation for details. + +The optional `IF NOT EXISTS` clause causes the error to be +suppressed if the branch already exists. + +The optional `WITH` clause can be used to set properties +on the newly created branch. + +The optional `FROM` clause can be used to set the source branch from which the +new branch is created. + +## Examples + +Create a new branch `audit` in the table `orders`: + +```sql +CREATE BRANCH audit IN TABLE orders +``` + +Create a new branch `audit` in the table `orders` from the branch `dev`: + +```sql +CREATE BRANCH audit IN TABLE orders FROM dev +``` + +## See also + +{doc}`drop-branch` diff --git a/docs/src/main/sphinx/sql/delete.md b/docs/src/main/sphinx/sql/delete.md index 622405faba92..aef7064f36f0 100644 --- a/docs/src/main/sphinx/sql/delete.md +++ b/docs/src/main/sphinx/sql/delete.md @@ -3,7 +3,7 @@ ## Synopsis ```text -DELETE FROM table_name [ WHERE condition ] +DELETE FROM table_name [ @ branch_name ] [ WHERE condition ] ``` ## Description @@ -32,6 +32,12 @@ Delete all orders: DELETE FROM orders; ``` +Delete all orders in the `audit` branch: + +```sql +DELETE FROM orders @ audit; +``` + ## Limitations Some connectors have limited or no support for `DELETE`. diff --git a/docs/src/main/sphinx/sql/deny.md b/docs/src/main/sphinx/sql/deny.md index d533382752db..68f12c667f5e 100644 --- a/docs/src/main/sphinx/sql/deny.md +++ b/docs/src/main/sphinx/sql/deny.md @@ -4,7 +4,7 @@ ```text DENY ( privilege [, ...] | ( ALL PRIVILEGES ) ) -ON ( table_name | TABLE table_name | SCHEMA schema_name) +ON [ BRANCH branch_name IN ] ( table_name | TABLE table_name | SCHEMA schema_name) TO ( user | USER user | ROLE role ) ``` @@ -39,6 +39,12 @@ Deny `SELECT` privilege on the table `orders` to everyone: DENY SELECT ON orders TO ROLE PUBLIC; ``` +Deny `INSERT` privilege on the `audit` branch of the `orders` table to user `alice`: + +```sql +DENY INSERT ON BRANCH audit IN orders TO alice; +``` + ## Limitations The system access controls as well as the connectors provided by default diff --git a/docs/src/main/sphinx/sql/drop-branch.md b/docs/src/main/sphinx/sql/drop-branch.md new file mode 100644 index 000000000000..4d29574c6fe0 --- /dev/null +++ b/docs/src/main/sphinx/sql/drop-branch.md @@ -0,0 +1,27 @@ +# DROP BRANCH + +## Synopsis + +```text +DROP BRANCH [ IF EXISTS ] branch_name +IN TABLE table_name +``` + +## Description + +Drops an existing branch. + +The optional `IF EXISTS` clause causes the error to be suppressed if the branch +does not exist. + +## Examples + +Drop the branch `audit` in the table `orders`: + +```sql +DROP BRANCH audit IN TABLE orders +``` + +## See also + +{doc}`create-branch` diff --git a/docs/src/main/sphinx/sql/grant.md b/docs/src/main/sphinx/sql/grant.md index 269e0fd9dda0..7c37831bf787 100644 --- a/docs/src/main/sphinx/sql/grant.md +++ b/docs/src/main/sphinx/sql/grant.md @@ -4,7 +4,7 @@ ```text GRANT ( privilege [, ...] | ( ALL PRIVILEGES ) ) -ON ( table_name | TABLE table_name | SCHEMA schema_name) +ON [ BRANCH branch_name IN ] ( table_name | TABLE table_name | SCHEMA schema_name) TO ( user | USER user | ROLE role ) [ WITH GRANT OPTION ] ``` @@ -51,6 +51,13 @@ Grant `SELECT` privilege on the table `orders` to everyone: GRANT SELECT ON orders TO ROLE PUBLIC; ``` +Grant `INSERT` privilege on the `audit` branch of the `orders` table to user +`alice`: + +```sql +GRANT INSERT ON BRANCH audit IN orders TO alice; +``` + ## Limitations Some connectors have no support for `GRANT`. diff --git a/docs/src/main/sphinx/sql/insert.md b/docs/src/main/sphinx/sql/insert.md index 677fc9a936fc..c4a0de948049 100644 --- a/docs/src/main/sphinx/sql/insert.md +++ b/docs/src/main/sphinx/sql/insert.md @@ -3,7 +3,7 @@ ## Synopsis ```text -INSERT INTO table_name [ ( column [, ... ] ) ] query +INSERT INTO table_name [ @ branch_name ] [ ( column [, ... ] ) ] query ``` ## Description @@ -52,6 +52,12 @@ INSERT INTO nation (nationkey, name, regionkey) VALUES (26, 'POLAND', 3); ``` +Insert a single row into `audit` branch of the `cities` table: + +```sql +INSERT INTO cities @ audit VALUES (1, 'San Francisco'); +``` + ## See also {doc}`values` diff --git a/docs/src/main/sphinx/sql/merge.md b/docs/src/main/sphinx/sql/merge.md index bec7422f2919..17e6a999816c 100644 --- a/docs/src/main/sphinx/sql/merge.md +++ b/docs/src/main/sphinx/sql/merge.md @@ -3,7 +3,7 @@ ## Synopsis ```text -MERGE INTO target_table [ [ AS ] target_alias ] +MERGE INTO target_table [ @ branch_name ] [ [ AS ] target_alias ] USING { source_table | query } [ [ AS ] source_alias ] ON search_condition when_clause [...] @@ -98,6 +98,15 @@ MERGE INTO accounts t USING monthly_accounts_update s VALUES(s.customer, s.purchases, s.address) ``` +Delete all customers mentioned in `audit` branch of the source table: + +```sql +MERGE INTO accounts @ audit t USING monthly_accounts_update s + ON t.customer = s.customer + WHEN MATCHED + THEN DELETE +``` + ## Limitations Any connector can be used as a source table for a `MERGE` statement. diff --git a/docs/src/main/sphinx/sql/revoke.md b/docs/src/main/sphinx/sql/revoke.md index 57fad9c7c59b..b93aaa08b702 100644 --- a/docs/src/main/sphinx/sql/revoke.md +++ b/docs/src/main/sphinx/sql/revoke.md @@ -5,7 +5,7 @@ ```text REVOKE [ GRANT OPTION FOR ] ( privilege [, ...] | ALL PRIVILEGES ) -ON ( table_name | TABLE table_name | SCHEMA schema_name ) +ON [ BRANCH branch_name IN ] ( table_name | TABLE table_name | SCHEMA schema_name ) FROM ( user | USER user | ROLE role ) ``` @@ -52,6 +52,13 @@ Revoke all privileges on the table `test` from user `alice`: REVOKE ALL PRIVILEGES ON test FROM alice; ``` +Revoke `INSERT` privilege on the `audit` branch of the `orders` table from user +`alice`: + +```sql +REVOKE INSERT ON BRANCH audit IN orders FROM alice; +``` + ## Limitations Some connectors have no support for `REVOKE`. diff --git a/docs/src/main/sphinx/sql/select.md b/docs/src/main/sphinx/sql/select.md index e863dc4bdd40..b50e66d29b8f 100644 --- a/docs/src/main/sphinx/sql/select.md +++ b/docs/src/main/sphinx/sql/select.md @@ -379,6 +379,32 @@ Complex grouping operations are often equivalent to a `UNION ALL` of simple does not apply, however, when the source of data for the aggregation is non-deterministic. +### AUTO + +When `AUTO` is specified, the Trino engine automatically determines the grouping +columns instead of requiring them to be listed explicitly. In this mode, any +column in the `SELECT` list that is not part of an aggregate function is +implicitly treated as a grouping column. + +This example query calculates the total account balance per market segment. +The `AUTO` clause derives `mktsegment` as the grouping key, since it is not used +in any aggregate function (i.e., `sum`). + +```sql +SELECT mktsegment, sum(acctbal) FROM shipping GROUP BY AUTO; +``` + +```text + mktsegment | _col1 +------------+-------------------- + BUILDING | 1444587.8 + MACHINERY | 1296958.61 + HOUSEHOLD | 1279340.66 + FURNITURE | 1265282.8 + AUTOMOBILE | 1395695.7200000004 +(5 rows) +``` + ### GROUPING SETS Grouping sets allow users to specify multiple lists of columns to group on. diff --git a/docs/src/main/sphinx/sql/show-branches.md b/docs/src/main/sphinx/sql/show-branches.md new file mode 100644 index 000000000000..bc68186e4e87 --- /dev/null +++ b/docs/src/main/sphinx/sql/show-branches.md @@ -0,0 +1,19 @@ +# SHOW BRANCHES + +## Synopsis + +```text +SHOW BRANCHES ( FROM | IN ) TABLE table_name +``` + +## Description + +List the available branches. + +## Examples + +List the branches in the table `orders`: + +```sql +SHOW BRANCHES IN TABLE orders +``` diff --git a/docs/src/main/sphinx/sql/update.md b/docs/src/main/sphinx/sql/update.md index 0f6a698f13ba..0daac81dae9c 100644 --- a/docs/src/main/sphinx/sql/update.md +++ b/docs/src/main/sphinx/sql/update.md @@ -3,7 +3,8 @@ ## Synopsis ```text -UPDATE table_name SET [ ( column = expression [, ... ] ) ] [ WHERE condition ] +UPDATE table_name [ @ branch_name ] +SET [ ( column = expression [, ... ] ) ] [ WHERE condition ] ``` ## Description @@ -55,6 +56,18 @@ SET ); ``` +Update the status of all purchases that haven't been assigned a ship date +in the `audit` branch: + +```sql +UPDATE + purchases @ audit +SET + status = 'OVERDUE' +WHERE + ship_date IS NULL; +``` + ## Limitations Some connectors have limited or no support for `UPDATE`. diff --git a/lib/trino-array/pom.xml b/lib/trino-array/pom.xml index c9f907a64ade..1b066416b1a1 100644 --- a/lib/trino-array/pom.xml +++ b/lib/trino-array/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-cache/pom.xml b/lib/trino-cache/pom.xml index 983b1716559e..6421ea9e027a 100644 --- a/lib/trino-cache/pom.xml +++ b/lib/trino-cache/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-alluxio/pom.xml b/lib/trino-filesystem-alluxio/pom.xml index 9263bdc85266..4c7e7516a247 100644 --- a/lib/trino-filesystem-alluxio/pom.xml +++ b/lib/trino-filesystem-alluxio/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -80,6 +80,12 @@ test + + io.trino + trino-plugin-toolkit + test + + io.trino trino-spi @@ -119,13 +125,13 @@ org.testcontainers - junit-jupiter + testcontainers test org.testcontainers - testcontainers + testcontainers-junit-jupiter test diff --git a/lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileSystem.java b/lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileSystem.java index 1bcf382648b4..f9cf67075b3e 100644 --- a/lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileSystem.java +++ b/lib/trino-filesystem-alluxio/src/test/java/io/trino/filesystem/alluxio/TestAlluxioFileSystem.java @@ -24,18 +24,15 @@ import io.trino.filesystem.AbstractTestTrinoFileSystem; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; +import io.trino.plugin.base.util.AutoCloseableCloser; import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.containers.Alluxio; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.utility.DockerImageName; import java.io.IOException; -import java.time.Duration; import static org.assertj.core.api.Assertions.assertThat; @@ -43,11 +40,7 @@ public class TestAlluxioFileSystem extends AbstractTestTrinoFileSystem { - private static final String IMAGE_NAME = "alluxio/alluxio:2.9.5"; - private static final DockerImageName ALLUXIO_IMAGE = DockerImageName.parse(IMAGE_NAME); - - private GenericContainer alluxioMaster; - private GenericContainer alluxioWorker; + private Alluxio alluxio; private TrinoFileSystem fileSystem; private Location rootLocation; private FileSystem alluxioFs; @@ -56,32 +49,33 @@ public class TestAlluxioFileSystem @BeforeAll void setup() { - alluxioMaster = createAlluxioMasterContainer(); - alluxioWorker = createAlluxioWorkerContainer(); + this.alluxio = new Alluxio(); + alluxio.start(); this.rootLocation = Location.of("alluxio:///"); InstancedConfiguration conf = Configuration.copyGlobal(); FileSystemContext fsContext = FileSystemContext.create(conf); this.alluxioFs = FileSystem.Factory.create(fsContext); this.alluxioFileSystemFactory = new AlluxioFileSystemFactory(conf); this.fileSystem = alluxioFileSystemFactory.create(ConnectorIdentity.ofUser("alluxio")); - // the SSHD container will be stopped by TestContainers on shutdown - // https://github.com/trinodb/trino/discussions/21969 - System.setProperty("ReportLeakedContainers.disabled", "true"); } @AfterAll void tearDown() + throws Exception { - if (alluxioMaster != null) { - alluxioMaster.close(); - } - if (alluxioWorker != null) { - alluxioWorker.close(); + try (AutoCloseableCloser closer = AutoCloseableCloser.create()) { + if (alluxio != null) { + closer.register(alluxio); + alluxio = null; + } + fileSystem = null; + if (alluxioFs != null) { + closer.register(alluxioFs); + alluxioFs = null; + } + rootLocation = null; + alluxioFileSystemFactory = null; } - fileSystem = null; - alluxioFs = null; - rootLocation = null; - alluxioFileSystemFactory = null; } @AfterEach @@ -137,45 +131,4 @@ protected boolean supportsIncompleteWriteNoClobber() { return false; } - - private static GenericContainer createAlluxioMasterContainer() - { - GenericContainer container = new GenericContainer<>(ALLUXIO_IMAGE); - container.withCommand("master-only") - .withEnv("ALLUXIO_JAVA_OPTS", - "-Dalluxio.security.authentication.type=NOSASL " - + "-Dalluxio.master.hostname=localhost " - + "-Dalluxio.worker.hostname=localhost " - + "-Dalluxio.master.mount.table.root.ufs=/opt/alluxio/underFSStorage " - + "-Dalluxio.master.journal.type=NOOP " - + "-Dalluxio.security.authorization.permission.enabled=false " - + "-Dalluxio.security.authorization.plugins.enabled=false ") - .withNetworkMode("host") - .withAccessToHost(true) - .waitingFor(new LogMessageWaitStrategy() - .withRegEx(".*Primary started*\n") - .withStartupTimeout(Duration.ofMinutes(3))); - container.start(); - return container; - } - - private static GenericContainer createAlluxioWorkerContainer() - { - GenericContainer container = new GenericContainer<>(ALLUXIO_IMAGE); - container.withCommand("worker-only") - .withNetworkMode("host") - .withEnv("ALLUXIO_JAVA_OPTS", - "-Dalluxio.security.authentication.type=NOSASL " - + "-Dalluxio.worker.ramdisk.size=128MB " - + "-Dalluxio.worker.hostname=localhost " - + "-Dalluxio.worker.tieredstore.level0.alias=HDD " - + "-Dalluxio.worker.tieredstore.level0.dirs.path=/tmp " - + "-Dalluxio.master.hostname=localhost " - + "-Dalluxio.security.authorization.permission.enabled=false " - + "-Dalluxio.security.authorization.plugins.enabled=false ") - .withAccessToHost(true) - .waitingFor(Wait.forLogMessage(".*Alluxio worker started.*\n", 1)); - container.start(); - return container; - } } diff --git a/lib/trino-filesystem-azure/pom.xml b/lib/trino-filesystem-azure/pom.xml index 3013d71abad2..a3a77345ee9b 100644 --- a/lib/trino-filesystem-azure/pom.xml +++ b/lib/trino-filesystem-azure/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -37,10 +37,6 @@ com.azure azure-identity - - com.nimbusds - oauth2-oidc-sdk - net.java.dev.jna jna-platform @@ -145,13 +141,6 @@ runtime - - com.nimbusds - oauth2-oidc-sdk - jdk11 - runtime - - io.airlift slice diff --git a/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureInputFile.java b/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureInputFile.java index 653cf0365baa..9c051785a810 100644 --- a/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureInputFile.java +++ b/lib/trino-filesystem-azure/src/main/java/io/trino/filesystem/azure/AzureInputFile.java @@ -25,6 +25,7 @@ import java.util.Optional; import java.util.OptionalLong; +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; import static io.trino.filesystem.azure.AzureUtils.handleAzureException; import static java.util.Objects.requireNonNull; @@ -101,6 +102,16 @@ public long length() return length.orElseThrow(); } + @Override + public String toString() + { + return toStringHelper(this) + .add("location", location) + .add("length", length) + .add("lastModified", lastModified) + .toString(); + } + private void loadProperties() throws IOException { diff --git a/lib/trino-filesystem-cache-alluxio/pom.xml b/lib/trino-filesystem-cache-alluxio/pom.xml index 3dea87128ec4..830bab5a3047 100644 --- a/lib/trino-filesystem-cache-alluxio/pom.xml +++ b/lib/trino-filesystem-cache-alluxio/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-gcs/pom.xml b/lib/trino-filesystem-gcs/pom.xml index 1ab64ea6df57..68f81f98461b 100644 --- a/lib/trino-filesystem-gcs/pom.xml +++ b/lib/trino-filesystem-gcs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java index 3e5020675655..1f94873a389e 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemConfig.java @@ -21,6 +21,7 @@ import io.airlift.units.Duration; import io.airlift.units.MinDuration; import jakarta.annotation.Nullable; +import jakarta.validation.constraints.AssertFalse; import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; @@ -33,6 +34,12 @@ public class GcsFileSystemConfig { + public enum AuthType + { + ACCESS_TOKEN, + SERVICE_ACCOUNT; + } + private DataSize readBlockSize = DataSize.of(2, MEGABYTE); private DataSize writeBlockSize = DataSize.of(16, MEGABYTE); private int pageSize = 100; @@ -41,7 +48,8 @@ public class GcsFileSystemConfig private String projectId; private Optional endpoint = Optional.empty(); - private boolean useGcsAccessToken; + private Optional useGcsAccessToken = Optional.empty(); + private Optional authType = Optional.empty(); private String jsonKey; private String jsonKeyFilePath; private int maxRetries = 20; @@ -134,15 +142,33 @@ public GcsFileSystemConfig setEndpoint(Optional endpoint) return this; } + @NotNull + public AuthType getAuthType() + { + if (useGcsAccessToken.isPresent() && useGcsAccessToken.get()) { + return AuthType.ACCESS_TOKEN; + } + return authType.orElse(AuthType.SERVICE_ACCOUNT); + } + + @Config("gcs.auth-type") + public GcsFileSystemConfig setAuthType(AuthType authType) + { + this.authType = Optional.of(authType); + return this; + } + + @Deprecated public boolean isUseGcsAccessToken() { - return useGcsAccessToken; + return useGcsAccessToken.orElse(false); } + @Deprecated @Config("gcs.use-access-token") public GcsFileSystemConfig setUseGcsAccessToken(boolean useGcsAccessToken) { - this.useGcsAccessToken = useGcsAccessToken; + this.useGcsAccessToken = Optional.of(useGcsAccessToken); return this; } @@ -268,13 +294,19 @@ public boolean isRetryDelayValid() return minBackoffDelay.compareTo(maxBackoffDelay) <= 0; } - @AssertTrue(message = "Either gcs.use-access-token or gcs.json-key or gcs.json-key-file-path must be set") + @AssertTrue(message = "Either gcs.auth-type or gcs.json-key or gcs.json-key-file-path must be set") public boolean isAuthMethodValid() { - if (useGcsAccessToken) { + if (getAuthType() == AuthType.ACCESS_TOKEN) { return jsonKey == null && jsonKeyFilePath == null; } return (jsonKey == null) ^ (jsonKeyFilePath == null); } + + @AssertFalse(message = "Cannot set both gcs.use-access-token and gcs.auth-type") + public boolean isAuthTypeAndGcsAccessTokenConfigured() + { + return authType.isPresent() && useGcsAccessToken.isPresent(); + } } diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java index e3eb7a469595..9e32abb19eed 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsFileSystemModule.java @@ -17,8 +17,8 @@ import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; -import static io.airlift.configuration.ConditionalModule.conditionalModule; import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.configuration.SwitchModule.switchModule; public class GcsFileSystemModule extends AbstractConfigurationAwareModule @@ -30,10 +30,12 @@ protected void setup(Binder binder) binder.bind(GcsStorageFactory.class).in(Scopes.SINGLETON); binder.bind(GcsFileSystemFactory.class).in(Scopes.SINGLETON); - install(conditionalModule( + install(switchModule( GcsFileSystemConfig.class, - GcsFileSystemConfig::isUseGcsAccessToken, - _ -> binder.bind(GcsAuth.class).to(GcsAccessTokenAuth.class).in(Scopes.SINGLETON), - _ -> binder.bind(GcsAuth.class).to(GcsDefaultAuth.class).in(Scopes.SINGLETON))); + GcsFileSystemConfig::getAuthType, + type -> switch (type) { + case ACCESS_TOKEN -> _ -> binder.bind(GcsAuth.class).to(GcsAccessTokenAuth.class).in(Scopes.SINGLETON); + case SERVICE_ACCOUNT -> _ -> binder.bind(GcsAuth.class).to(GcsServiceAccountAuth.class).in(Scopes.SINGLETON); + })); } } diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsInputFile.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsInputFile.java index 16b3e0f3eaa0..e3898ea8c469 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsInputFile.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsInputFile.java @@ -26,6 +26,7 @@ import java.util.Optional; import java.util.OptionalLong; +import static com.google.common.base.MoreObjects.toStringHelper; import static io.trino.filesystem.gcs.GcsUtils.encodedKey; import static io.trino.filesystem.gcs.GcsUtils.getBlob; import static io.trino.filesystem.gcs.GcsUtils.getBlobOrThrow; @@ -107,6 +108,17 @@ public Location location() return location.location(); } + @Override + public String toString() + { + return toStringHelper(this) + .add("location", location) + .add("predeclaredLength", predeclaredLength) + .add("length", length) + .add("lastModified", lastModified) + .toString(); + } + private void loadProperties() throws IOException { diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsOutputStream.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsOutputStream.java index a931e24cf849..d13267966148 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsOutputStream.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsOutputStream.java @@ -141,6 +141,7 @@ public void close() if (e.getCode() == HTTP_PRECON_FAILED) { throw new FileAlreadyExistsException(location.toString()); } + throw new IOException(e); } catch (IOException e) { throw new IOException("Error closing file: " + location, e); diff --git a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsDefaultAuth.java b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsServiceAccountAuth.java similarity index 96% rename from lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsDefaultAuth.java rename to lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsServiceAccountAuth.java index d5935b437508..a30ddacf0942 100644 --- a/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsDefaultAuth.java +++ b/lib/trino-filesystem-gcs/src/main/java/io/trino/filesystem/gcs/GcsServiceAccountAuth.java @@ -26,13 +26,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; -public class GcsDefaultAuth +public class GcsServiceAccountAuth implements GcsAuth { private final Optional jsonGoogleCredential; @Inject - public GcsDefaultAuth(GcsFileSystemConfig config) + public GcsServiceAccountAuth(GcsFileSystemConfig config) throws IOException { String jsonKey = config.getJsonKey(); diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java index 6e54f1ed7c55..6c5c1d93baa8 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/AbstractTestGcsFileSystem.java @@ -56,7 +56,7 @@ protected void initialize(String gcpCredentialKey) // For gcp testing this corresponds to the Cluster Storage Admin and Cluster Storage Object Admin roles byte[] jsonKeyBytes = Base64.getDecoder().decode(gcpCredentialKey); GcsFileSystemConfig config = new GcsFileSystemConfig().setJsonKey(new String(jsonKeyBytes, UTF_8)); - GcsStorageFactory storageFactory = new GcsStorageFactory(config, new GcsDefaultAuth(config)); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new GcsServiceAccountAuth(config)); this.gcsFileSystemFactory = new GcsFileSystemFactory(config, storageFactory); this.storage = storageFactory.create(ConnectorIdentity.ofUser("test")); String bucket = RemoteStorageHelper.generateBucketName(); diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java index 05a2259fb77e..179bce9cb78c 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsFileSystemConfig.java @@ -16,6 +16,8 @@ import com.google.common.collect.ImmutableMap; import io.airlift.units.DataSize; import io.airlift.units.Duration; +import io.trino.filesystem.gcs.GcsFileSystemConfig.AuthType; +import jakarta.validation.constraints.AssertFalse; import jakarta.validation.constraints.AssertTrue; import org.junit.jupiter.api.Test; @@ -41,9 +43,10 @@ void testDefaults() .setWriteBlockSize(DataSize.of(16, MEGABYTE)) .setPageSize(100) .setBatchSize(100) + .setUseGcsAccessToken(false) .setProjectId(null) .setEndpoint(Optional.empty()) - .setUseGcsAccessToken(false) + .setAuthType(AuthType.SERVICE_ACCOUNT) .setJsonKey(null) .setJsonKeyFilePath(null) .setMaxRetries(20) @@ -56,6 +59,43 @@ void testDefaults() @Test void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("gcs.read-block-size", "51MB") + .put("gcs.write-block-size", "52MB") + .put("gcs.page-size", "10") + .put("gcs.batch-size", "11") + .put("gcs.project-id", "project") + .put("gcs.endpoint", "http://custom.dns.org:8000") + .put("gcs.auth-type", "access_token") + .put("gcs.client.max-retries", "10") + .put("gcs.client.backoff-scale-factor", "4.0") + .put("gcs.client.max-retry-time", "10s") + .put("gcs.client.min-backoff-delay", "20ms") + .put("gcs.client.max-backoff-delay", "20ms") + .put("gcs.application-id", "application id") + .buildOrThrow(); + + GcsFileSystemConfig expected = new GcsFileSystemConfig() + .setReadBlockSize(DataSize.of(51, MEGABYTE)) + .setWriteBlockSize(DataSize.of(52, MEGABYTE)) + .setPageSize(10) + .setBatchSize(11) + .setProjectId("project") + .setEndpoint(Optional.of("http://custom.dns.org:8000")) + .setAuthType(AuthType.ACCESS_TOKEN) + .setMaxRetries(10) + .setBackoffScaleFactor(4.0) + .setMaxRetryTime(new Duration(10, SECONDS)) + .setMinBackoffDelay(new Duration(20, MILLISECONDS)) + .setMaxBackoffDelay(new Duration(20, MILLISECONDS)) + .setApplicationId("application id"); + assertFullMapping(properties, expected, Set.of("gcs.json-key", "gcs.json-key-file-path", "gcs.use-access-token")); + } + + // backwards compatibility test, remove if use-access-token is removed + @Test + void testExplicitPropertyMappingsWithDeprecatedUseAccessToken() { Map properties = ImmutableMap.builder() .put("gcs.read-block-size", "51MB") @@ -87,7 +127,7 @@ void testExplicitPropertyMappings() .setMinBackoffDelay(new Duration(20, MILLISECONDS)) .setMaxBackoffDelay(new Duration(20, MILLISECONDS)) .setApplicationId("application id"); - assertFullMapping(properties, expected, Set.of("gcs.json-key", "gcs.json-key-file-path")); + assertFullMapping(properties, expected, Set.of("gcs.json-key", "gcs.json-key-file-path", "gcs.auth-type")); } @Test @@ -95,18 +135,18 @@ public void testValidation() { assertFailsValidation( new GcsFileSystemConfig() - .setUseGcsAccessToken(true) + .setAuthType(AuthType.ACCESS_TOKEN) .setJsonKey("{}}"), "authMethodValid", - "Either gcs.use-access-token or gcs.json-key or gcs.json-key-file-path must be set", + "Either gcs.auth-type or gcs.json-key or gcs.json-key-file-path must be set", AssertTrue.class); assertFailsValidation( new GcsFileSystemConfig() - .setUseGcsAccessToken(true) + .setAuthType(AuthType.ACCESS_TOKEN) .setJsonKeyFilePath("/dev/null"), "authMethodValid", - "Either gcs.use-access-token or gcs.json-key or gcs.json-key-file-path must be set", + "Either gcs.auth-type or gcs.json-key or gcs.json-key-file-path must be set", AssertTrue.class); assertFailsValidation( @@ -114,7 +154,7 @@ public void testValidation() .setJsonKey("{}") .setJsonKeyFilePath("/dev/null"), "authMethodValid", - "Either gcs.use-access-token or gcs.json-key or gcs.json-key-file-path must be set", + "Either gcs.auth-type or gcs.json-key or gcs.json-key-file-path must be set", AssertTrue.class); assertFailsValidation( @@ -125,5 +165,37 @@ public void testValidation() "retryDelayValid", "gcs.client.min-backoff-delay must be less than or equal to gcs.client.max-backoff-delay", AssertTrue.class); + + assertFailsValidation( + new GcsFileSystemConfig() + .setAuthType(AuthType.ACCESS_TOKEN) + .setUseGcsAccessToken(true), + "authTypeAndGcsAccessTokenConfigured", + "Cannot set both gcs.use-access-token and gcs.auth-type", + AssertFalse.class); + + assertFailsValidation( + new GcsFileSystemConfig() + .setAuthType(AuthType.ACCESS_TOKEN) + .setUseGcsAccessToken(false), + "authTypeAndGcsAccessTokenConfigured", + "Cannot set both gcs.use-access-token and gcs.auth-type", + AssertFalse.class); + + assertFailsValidation( + new GcsFileSystemConfig() + .setUseGcsAccessToken(true) + .setAuthType(AuthType.SERVICE_ACCOUNT), + "authTypeAndGcsAccessTokenConfigured", + "Cannot set both gcs.use-access-token and gcs.auth-type", + AssertFalse.class); + + assertFailsValidation( + new GcsFileSystemConfig() + .setUseGcsAccessToken(false) + .setAuthType(AuthType.SERVICE_ACCOUNT), + "authTypeAndGcsAccessTokenConfigured", + "Cannot set both gcs.use-access-token and gcs.auth-type", + AssertFalse.class); } } diff --git a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java index 2edc89fb6b04..89247ee7334e 100644 --- a/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java +++ b/lib/trino-filesystem-gcs/src/test/java/io/trino/filesystem/gcs/TestGcsStorageFactory.java @@ -32,7 +32,7 @@ void testDefaultCredentials() // No credentials options are set GcsFileSystemConfig config = new GcsFileSystemConfig(); - GcsStorageFactory storageFactory = new GcsStorageFactory(config, new GcsDefaultAuth(config)); + GcsStorageFactory storageFactory = new GcsStorageFactory(config, new GcsServiceAccountAuth(config)); Credentials actualCredentials; try (Storage storage = storageFactory.create(ConnectorIdentity.ofUser("test"))) { diff --git a/lib/trino-filesystem-manager/pom.xml b/lib/trino-filesystem-manager/pom.xml index 005b3d47eed5..df2c2738264e 100644 --- a/lib/trino-filesystem-manager/pom.xml +++ b/lib/trino-filesystem-manager/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java b/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java index 9d26955feb0f..411076795704 100644 --- a/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java +++ b/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/FileSystemModule.java @@ -20,7 +20,6 @@ import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.configuration.ConfigPropertyMetadata; -import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystemFactory; @@ -46,6 +45,7 @@ import io.trino.filesystem.switching.SwitchingFileSystemFactory; import io.trino.filesystem.tracing.TracingFileSystemFactory; import io.trino.filesystem.tracking.TrackingFileSystemFactory; +import io.trino.spi.connector.ConnectorContext; import java.util.Map; import java.util.Optional; @@ -60,15 +60,15 @@ public class FileSystemModule extends AbstractConfigurationAwareModule { private final String catalogName; + private final ConnectorContext context; private final boolean isCoordinator; - private final OpenTelemetry openTelemetry; private final boolean coordinatorFileCaching; - public FileSystemModule(String catalogName, boolean isCoordinator, OpenTelemetry openTelemetry, boolean coordinatorFileCaching) + public FileSystemModule(String catalogName, ConnectorContext context, boolean coordinatorFileCaching) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); - this.isCoordinator = isCoordinator; - this.openTelemetry = requireNonNull(openTelemetry, "openTelemetry is null"); + this.context = requireNonNull(context, "context is null"); + this.isCoordinator = context.getCurrentNode().isCoordinator(); this.coordinatorFileCaching = coordinatorFileCaching; } @@ -86,7 +86,7 @@ protected void setup(Binder binder) !config.isNativeGcsEnabled(), !config.isNativeS3Enabled(), catalogName, - openTelemetry); + context); loader.configure().forEach((name, securitySensitive) -> consumeProperty(new ConfigPropertyMetadata(name, securitySensitive))); diff --git a/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/HdfsFileSystemLoader.java b/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/HdfsFileSystemLoader.java index 038a9f69e6df..19b1c470eced 100644 --- a/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/HdfsFileSystemLoader.java +++ b/lib/trino-filesystem-manager/src/main/java/io/trino/filesystem/manager/HdfsFileSystemLoader.java @@ -13,10 +13,10 @@ */ package io.trino.filesystem.manager; -import io.opentelemetry.api.OpenTelemetry; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.spi.Plugin; import io.trino.spi.classloader.ThreadContextClassLoader; +import io.trino.spi.connector.ConnectorContext; import jakarta.annotation.PreDestroy; import java.io.File; @@ -45,7 +45,7 @@ public HdfsFileSystemLoader( boolean gcsEnabled, boolean s3Enabled, String catalogName, - OpenTelemetry openTelemetry) + ConnectorContext context) { Class clazz = tryLoadExistingHdfsManager(); @@ -73,8 +73,8 @@ public HdfsFileSystemLoader( } try (var _ = new ThreadContextClassLoader(classLoader)) { - manager = clazz.getConstructor(Map.class, boolean.class, boolean.class, boolean.class, String.class, OpenTelemetry.class) - .newInstance(config, azureEnabled, gcsEnabled, s3Enabled, catalogName, openTelemetry); + manager = clazz.getConstructor(Map.class, boolean.class, boolean.class, boolean.class, String.class, ConnectorContext.class) + .newInstance(config, azureEnabled, gcsEnabled, s3Enabled, catalogName, context); } catch (ReflectiveOperationException e) { throw new RuntimeException(e); diff --git a/lib/trino-filesystem-s3/pom.xml b/lib/trino-filesystem-s3/pom.xml index 77fee4f9864a..fa47189ccb04 100644 --- a/lib/trino-filesystem-s3/pom.xml +++ b/lib/trino-filesystem-s3/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -272,25 +272,25 @@ org.testcontainers - junit-jupiter + testcontainers test org.testcontainers - localstack + testcontainers-junit-jupiter test org.testcontainers - testcontainers + testcontainers-localstack test org.testcontainers - toxiproxy + testcontainers-toxiproxy test diff --git a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputFile.java b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputFile.java index f9a2f77f41e0..4c92ef348ddb 100644 --- a/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputFile.java +++ b/lib/trino-filesystem-s3/src/main/java/io/trino/filesystem/s3/S3InputFile.java @@ -32,6 +32,7 @@ import java.time.Instant; import java.util.Optional; +import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Verify.verify; import static io.trino.filesystem.s3.S3FileSystemConfig.S3SseType.NONE; import static io.trino.filesystem.s3.S3SseCUtils.encoded; @@ -109,6 +110,16 @@ public Location location() return location.location(); } + @Override + public String toString() + { + return toStringHelper(this) + .add("location", location) + .add("length", length) + .add("lastModified", lastModified) + .toString(); + } + private GetObjectRequest newGetObjectRequest() { return GetObjectRequest.builder() diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemLocalStack.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemLocalStack.java index 796640b067ff..b0303bf37db6 100644 --- a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemLocalStack.java +++ b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3FileSystemLocalStack.java @@ -16,10 +16,9 @@ import io.airlift.units.DataSize; import io.opentelemetry.api.OpenTelemetry; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.localstack.LocalStackContainer; -import org.testcontainers.containers.localstack.LocalStackContainer.Service; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.localstack.LocalStackContainer; import org.testcontainers.utility.DockerImageName; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -36,7 +35,7 @@ public class TestS3FileSystemLocalStack @Container private static final LocalStackContainer LOCALSTACK = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.0.3")) - .withServices(Service.S3); + .withServices("s3"); @Override protected void initEnvironment() @@ -56,7 +55,7 @@ protected String bucket() protected S3Client createS3Client() { return S3Client.builder() - .endpointOverride(LOCALSTACK.getEndpointOverride(Service.S3)) + .endpointOverride(LOCALSTACK.getEndpoint()) .region(Region.of(LOCALSTACK.getRegion())) .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create(LOCALSTACK.getAccessKey(), LOCALSTACK.getSecretKey()))) @@ -69,7 +68,7 @@ protected S3FileSystemFactory createS3FileSystemFactory() return new S3FileSystemFactory(OpenTelemetry.noop(), new S3FileSystemConfig() .setAwsAccessKey(LOCALSTACK.getAccessKey()) .setAwsSecretKey(LOCALSTACK.getSecretKey()) - .setEndpoint(LOCALSTACK.getEndpointOverride(Service.S3).toString()) + .setEndpoint(LOCALSTACK.getEndpoint().toString()) .setRegion(LOCALSTACK.getRegion()) .setStreamingPartSize(DataSize.valueOf("5.5MB")), new S3FileSystemStats()); } diff --git a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3Retries.java b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3Retries.java index 37b5551e0117..e82522b8dbde 100644 --- a/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3Retries.java +++ b/lib/trino-filesystem-s3/src/test/java/io/trino/filesystem/s3/TestS3Retries.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.testcontainers.containers.Network; -import org.testcontainers.containers.ToxiproxyContainer; +import org.testcontainers.toxiproxy.ToxiproxyContainer; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.exception.SdkClientException; diff --git a/lib/trino-filesystem/pom.xml b/lib/trino-filesystem/pom.xml index 95910788af19..940f1305d9f7 100644 --- a/lib/trino-filesystem/pom.xml +++ b/lib/trino-filesystem/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-filesystem/src/main/java/io/trino/filesystem/TrinoInput.java b/lib/trino-filesystem/src/main/java/io/trino/filesystem/TrinoInput.java index 30be80b84d7a..08b3053b038b 100644 --- a/lib/trino-filesystem/src/main/java/io/trino/filesystem/TrinoInput.java +++ b/lib/trino-filesystem/src/main/java/io/trino/filesystem/TrinoInput.java @@ -18,17 +18,52 @@ import io.trino.spi.metrics.Metrics; import java.io.Closeable; +import java.io.EOFException; import java.io.IOException; public interface TrinoInput extends Closeable { - void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength) + /** + * Read exactly {@code length} bytes starting from input position {@code position} + * into the provided buffer starting at offset {@code bufferOffset}. + * The buffer positions {@code [0..bufferOffset)} are not modified. + * + * @param position the position in the input to start reading from; must be non-negative + * @param buffer the buffer to read bytes into; must be non-null + * @param bufferOffset the offset in the buffer to start writing bytes to; must be non-negative + * @param length the number of bytes to read; must be non-negative + * @throws IndexOutOfBoundsException when {@code length} or {@code bufferOffset} is negative, + * or when {@code bufferOffset + length} is greater than {@code buffer.length} + * @throws EOFException when input has fewer than {@code position + length} bytes + * @throws IOException when {@code position} is negative, or when any other I/O error occurs + */ + void readFully(long position, byte[] buffer, int bufferOffset, int length) throws IOException; - int readTail(byte[] buffer, int bufferOffset, int bufferLength) + /** + * Read at most {@code maxLength} bytes from the end of the input into the provided buffer + * starting at offset {@code bufferOffset}. The buffer positions {@code [0..bufferOffset)} + * are not modified. + * + * @param buffer the buffer to read bytes into; must be non-null + * @param bufferOffset the offset in the buffer to start writing bytes to; must be non-negative + * @param maxLength the maximum number of bytes to read; must be non-negative + * @return the number of bytes actually read, which may be less than {@code maxLength} if + * the input contains fewer bytes than {@code maxLength} + */ + int readTail(byte[] buffer, int bufferOffset, int maxLength) throws IOException; + /** + * Read exactly {@code length} bytes starting from input position {@code position} + * and return them as a Slice. + * + * @param position the position in the input to start reading from; must be non-negative + * @param length the number of bytes to read; must be non-negative + * @throws EOFException when input has fewer than {@code position + length} bytes + * @throws IOException when {@code position} is negative, or when any other I/O error occurs + */ default Slice readFully(long position, int length) throws IOException { @@ -37,6 +72,9 @@ default Slice readFully(long position, int length) return Slices.wrappedBuffer(buffer); } + /** + * Read at most {@code length} bytes from the end of the input and return them as a Slice. + */ default Slice readTail(int length) throws IOException { diff --git a/lib/trino-filesystem/src/main/java/io/trino/filesystem/cache/CacheInputFile.java b/lib/trino-filesystem/src/main/java/io/trino/filesystem/cache/CacheInputFile.java index 3c7f6e1fe3d2..4db7ed718f69 100644 --- a/lib/trino-filesystem/src/main/java/io/trino/filesystem/cache/CacheInputFile.java +++ b/lib/trino-filesystem/src/main/java/io/trino/filesystem/cache/CacheInputFile.java @@ -23,6 +23,7 @@ import java.util.Optional; import java.util.OptionalLong; +import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; public final class CacheInputFile @@ -103,4 +104,14 @@ public Location location() { return delegate.location(); } + + @Override + public String toString() + { + return toStringHelper(this) + .add("delegate", delegate) + .add("length", length) + .add("lastModified", lastModified) + .toString(); + } } diff --git a/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracing/TracingInputFile.java b/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracing/TracingInputFile.java index 69415c5da8f5..9e24ade0961e 100644 --- a/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracing/TracingInputFile.java +++ b/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracing/TracingInputFile.java @@ -119,6 +119,6 @@ public Location location() @Override public String toString() { - return location().toString(); + return delegate.toString(); } } diff --git a/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracking/TrackingInputFile.java b/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracking/TrackingInputFile.java index 1727560fee2d..e7170d3eb5b3 100644 --- a/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracking/TrackingInputFile.java +++ b/lib/trino-filesystem/src/main/java/io/trino/filesystem/tracking/TrackingInputFile.java @@ -76,4 +76,10 @@ public Location location() { return delegate.location(); } + + @Override + public String toString() + { + return delegate.toString(); + } } diff --git a/lib/trino-filesystem/src/test/java/io/trino/filesystem/AbstractTestTrinoFileSystem.java b/lib/trino-filesystem/src/test/java/io/trino/filesystem/AbstractTestTrinoFileSystem.java index 29febe20c03f..245159e053ac 100644 --- a/lib/trino-filesystem/src/test/java/io/trino/filesystem/AbstractTestTrinoFileSystem.java +++ b/lib/trino-filesystem/src/test/java/io/trino/filesystem/AbstractTestTrinoFileSystem.java @@ -1047,19 +1047,24 @@ public void testPreSignedUris() } Optional directLocation = getFileSystem() - .preSignedUri(location, new Duration(3, SECONDS)); + .preSignedUri(location, new Duration(30, SECONDS)); + + Optional expiredDirectLocation = getFileSystem() + .preSignedUri(location, new Duration(1, SECONDS)); assertThat(directLocation).isPresent(); - assertThat(retrieveUri(directLocation.get())) - .isEqualTo(TEST_BLOB_CONTENT_PREFIX + location); + + assertEventually(new Duration(5, SECONDS), () -> assertThat(retrieveUri(directLocation.get())) + .isEqualTo(TEST_BLOB_CONTENT_PREFIX + location)); // Check if it can be retrieved more than once - assertThat(retrieveUri(directLocation.get())) - .isEqualTo(TEST_BLOB_CONTENT_PREFIX + location); + assertEventually(new Duration(5, SECONDS), () -> assertThat(retrieveUri(directLocation.get())) + .isEqualTo(TEST_BLOB_CONTENT_PREFIX + location)); // Check if after a timeout the pre-signed URI is no longer valid - assertEventually(new Duration(5, SECONDS), new Duration(1, SECONDS), () -> assertThatThrownBy(() -> retrieveUri(directLocation.get())) - .isInstanceOf(IOException.class)); + assertEventually(new Duration(5, SECONDS), new Duration(1, SECONDS), () -> assertThatThrownBy(() -> retrieveUri(expiredDirectLocation.get())) + .isInstanceOf(IOException.class) + .hasMessageContaining("Failed to retrieve")); } } diff --git a/lib/trino-geospatial-toolkit/pom.xml b/lib/trino-geospatial-toolkit/pom.xml index a3f744f9c964..4e2faf107806 100644 --- a/lib/trino-geospatial-toolkit/pom.xml +++ b/lib/trino-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-hdfs/pom.xml b/lib/trino-hdfs/pom.xml index 9faa3859b620..6845a28aea67 100644 --- a/lib/trino-hdfs/pom.xml +++ b/lib/trino-hdfs/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -254,13 +254,13 @@ org.testcontainers - junit-jupiter + testcontainers test org.testcontainers - testcontainers + testcontainers-junit-jupiter test @@ -292,6 +292,20 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsFileSystemManager.java b/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsFileSystemManager.java index 4f01dd09c248..44fff919c843 100644 --- a/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsFileSystemManager.java +++ b/lib/trino-hdfs/src/main/java/io/trino/filesystem/hdfs/HdfsFileSystemManager.java @@ -18,7 +18,6 @@ import io.airlift.bootstrap.Bootstrap; import io.airlift.bootstrap.LifeCycleManager; import io.airlift.configuration.ConfigPropertyMetadata; -import io.opentelemetry.api.OpenTelemetry; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.hdfs.HdfsModule; import io.trino.hdfs.authentication.HdfsAuthenticationModule; @@ -26,9 +25,10 @@ import io.trino.hdfs.cos.HiveCosModule; import io.trino.hdfs.gcs.HiveGcsModule; import io.trino.hdfs.s3.HiveS3Module; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.spi.catalog.CatalogName; +import io.trino.spi.connector.ConnectorContext; import org.weakref.jmx.guice.MBeanModule; import java.util.ArrayList; @@ -49,7 +49,7 @@ public HdfsFileSystemManager( boolean gcsEnabled, boolean s3Enabled, String catalogName, - OpenTelemetry openTelemetry) + ConnectorContext context) { List modules = new ArrayList<>(); @@ -61,10 +61,7 @@ public HdfsFileSystemManager( modules.add(new HdfsModule()); modules.add(new HdfsAuthenticationModule()); modules.add(new HiveCosModule()); - modules.add(binder -> { - binder.bind(OpenTelemetry.class).toInstance(openTelemetry); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }); + modules.add(new ConnectorContextModule(catalogName, context)); if (azureEnabled) { modules.add(new HiveAzureModule()); diff --git a/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/TestHdfsFileSystemManager.java b/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/TestHdfsFileSystemManager.java index 82567751f9f0..9e6637cd9a51 100644 --- a/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/TestHdfsFileSystemManager.java +++ b/lib/trino-hdfs/src/test/java/io/trino/filesystem/hdfs/TestHdfsFileSystemManager.java @@ -14,11 +14,11 @@ package io.trino.filesystem.hdfs; import com.google.common.collect.ImmutableMap; -import io.opentelemetry.api.OpenTelemetry; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.spi.security.ConnectorIdentity; +import io.trino.testing.TestingConnectorContext; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -42,7 +42,7 @@ void testManager() true, true, "test", - OpenTelemetry.noop()); + new TestingConnectorContext()); assertThat(manager.configure().keySet()).containsExactly("hive.dfs.verify-checksum", "hive.s3.region"); diff --git a/lib/trino-hive-formats/pom.xml b/lib/trino-hive-formats/pom.xml index 697bac77140c..d925d79e024e 100644 --- a/lib/trino-hive-formats/pom.xml +++ b/lib/trino-hive-formats/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java index dc603c3c2372..7df72f341966 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java @@ -256,7 +256,7 @@ public static ValidateLogicalTypeResult validateLogicalType(Schema schema) case DATE, DECIMAL, TIME_MILLIS, TIME_MICROS, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, UUID: logicalType = fromSchemaIgnoreInvalid(schema); break; - case LOCAL_TIMESTAMP_MICROS + LOCAL_TIMESTAMP_MILLIS: + case LOCAL_TIMESTAMP_MICROS, LOCAL_TIMESTAMP_MILLIS: log.debug("Logical type %s not currently supported by by Trino", typeName); // fall through default: diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/encodings/text/MapEncoding.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/encodings/text/MapEncoding.java index 9006bd3325ce..0a4d3c8e25c3 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/encodings/text/MapEncoding.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/encodings/text/MapEncoding.java @@ -205,7 +205,7 @@ public void decodeKeyValue(int depth, Slice slice, int keyOffset, int keyLength, throws FileCorruptionException { if (distinctKeys[entryPosition]) { - mapType.getKeyType().appendTo(keyBlock, entryPosition, keyBuilder); + keyBuilder.append(keyBlock.getUnderlyingValueBlock(), keyBlock.getUnderlyingValuePosition(entryPosition)); if (hasValue && !isNullSequence(slice, valueOffset, valueLength)) { valueEncoding.decodeValueInto(valueBuilder, slice, valueOffset, valueLength); diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonDeserializer.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonDeserializer.java index 85ba728d2ecc..2524095b3698 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonDeserializer.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonDeserializer.java @@ -772,7 +772,7 @@ void decodeValue(Object jsonValue, BlockBuilder builder) int keyIndex = 0; for (Object fieldName : fieldNames) { if (distinctKeys[keyIndex]) { - keyType.appendTo(keyBlock, keyIndex, keyBuilder); + keyBuilder.append(keyBlock.getUnderlyingValueBlock(), keyBlock.getUnderlyingValuePosition(keyIndex)); valueDecoder.decode(jsonObject.get(fieldName), valueBuilder); } keyIndex++; diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonSerializer.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonSerializer.java index 73b38b8da521..d811bbf12b87 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonSerializer.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/line/openxjson/OpenXJsonSerializer.java @@ -269,18 +269,18 @@ public static boolean isSupportedType(Type type) } return BOOLEAN.equals(type) || - BIGINT.equals(type) || - INTEGER.equals(type) || - SMALLINT.equals(type) || - TINYINT.equals(type) || - type instanceof DecimalType || - REAL.equals(type) || - DOUBLE.equals(type) || - DATE.equals(type) || - type instanceof TimestampType || - VARBINARY.equals(type) || - type instanceof VarcharType || - type instanceof CharType; + BIGINT.equals(type) || + INTEGER.equals(type) || + SMALLINT.equals(type) || + TINYINT.equals(type) || + type instanceof DecimalType || + REAL.equals(type) || + DOUBLE.equals(type) || + DATE.equals(type) || + type instanceof TimestampType || + VARBINARY.equals(type) || + type instanceof VarcharType || + type instanceof CharType; } private static boolean isStructuralType(Type type) diff --git a/lib/trino-matching/pom.xml b/lib/trino-matching/pom.xml index 7b9914ea0c32..2afa16c64bfb 100644 --- a/lib/trino-matching/pom.xml +++ b/lib/trino-matching/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-matching/src/main/java/io/trino/matching/Capture.java b/lib/trino-matching/src/main/java/io/trino/matching/Capture.java index 6e8ccf8f18ec..91a2846f12c3 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/Capture.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/Capture.java @@ -15,11 +15,16 @@ import java.util.concurrent.atomic.AtomicInteger; -public class Capture +import static java.util.Objects.requireNonNull; + +public record Capture(String description) { private static final AtomicInteger sequenceCounter = new AtomicInteger(); - private final String description; + public Capture + { + requireNonNull(description, "description is null"); + } public static Capture newCapture() { @@ -30,14 +35,4 @@ public static Capture newCapture(String description) { return new Capture<>(description + "@" + sequenceCounter.incrementAndGet()); } - - private Capture(String description) - { - this.description = description; - } - - public String description() - { - return description; - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/pattern/CapturePattern.java b/lib/trino-matching/src/main/java/io/trino/matching/CapturePattern.java similarity index 76% rename from lib/trino-matching/src/main/java/io/trino/matching/pattern/CapturePattern.java rename to lib/trino-matching/src/main/java/io/trino/matching/CapturePattern.java index 2d3e577228fe..2017d5db6866 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/pattern/CapturePattern.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/CapturePattern.java @@ -11,19 +11,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.matching.pattern; - -import io.trino.matching.Capture; -import io.trino.matching.Captures; -import io.trino.matching.Match; -import io.trino.matching.Pattern; -import io.trino.matching.PatternVisitor; +package io.trino.matching; import java.util.stream.Stream; import static java.util.Objects.requireNonNull; -public class CapturePattern +public final class CapturePattern extends Pattern { private final Capture capture; @@ -45,10 +39,4 @@ public Stream accept(Object object, Captures captures, C context) Captures newCaptures = captures.addAll(Captures.ofNullable(capture, (T) object)); return Stream.of(Match.of(newCaptures)); } - - @Override - public void accept(PatternVisitor patternVisitor) - { - patternVisitor.visitCapture(this); - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/Captures.java b/lib/trino-matching/src/main/java/io/trino/matching/Captures.java index 7880218ee042..9fc8654f5d7c 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/Captures.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/Captures.java @@ -14,23 +14,11 @@ package io.trino.matching; import java.util.NoSuchElementException; -import java.util.Objects; -public class Captures +public record Captures(Capture capture, Object value, Captures tail) { private static final Captures NIL = new Captures(null, null, null); - private final Capture capture; - private final Object value; - private final Captures tail; - - private Captures(Capture capture, Object value, Captures tail) - { - this.capture = capture; - this.value = value; - this.tail = tail; - } - public static Captures empty() { return NIL; @@ -60,29 +48,4 @@ public T get(Capture capture) } return tail.get(capture); } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Captures captures = (Captures) o; - return Objects.equals(capture, captures.capture) - && Objects.equals(value, captures.value) - && Objects.equals(tail, captures.tail); - } - - @Override - public int hashCode() - { - int result = capture != null ? capture.hashCode() : 0; - result = 31 * result + (value != null ? value.hashCode() : 0); - result = 31 * result + (tail != null ? tail.hashCode() : 0); - return result; - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/CustomPattern.java b/lib/trino-matching/src/main/java/io/trino/matching/CustomPattern.java new file mode 100644 index 000000000000..52b902a0b4c1 --- /dev/null +++ b/lib/trino-matching/src/main/java/io/trino/matching/CustomPattern.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.matching; + +import java.util.Optional; + +public abstract non-sealed class CustomPattern + extends Pattern +{ + protected CustomPattern(Pattern previous) + { + super(previous); + } + + public CustomPattern(Optional> previous) + { + super(previous); + } + + public abstract String print(); +} diff --git a/lib/trino-matching/src/main/java/io/trino/matching/DefaultPrinter.java b/lib/trino-matching/src/main/java/io/trino/matching/DefaultPrinter.java deleted file mode 100644 index 42e22ded9699..000000000000 --- a/lib/trino-matching/src/main/java/io/trino/matching/DefaultPrinter.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.matching; - -import io.trino.matching.pattern.CapturePattern; -import io.trino.matching.pattern.EqualsPattern; -import io.trino.matching.pattern.FilterPattern; -import io.trino.matching.pattern.OrPattern; -import io.trino.matching.pattern.TypeOfPattern; -import io.trino.matching.pattern.WithPattern; - -import java.util.Iterator; - -import static java.lang.String.format; - -public class DefaultPrinter - implements PatternVisitor -{ - private final StringBuilder result = new StringBuilder(); - private int level; - - public String result() - { - return result.toString(); - } - - @Override - public void visitTypeOf(TypeOfPattern pattern) - { - visitPrevious(pattern); - appendLine("typeOf(%s)", pattern.expectedClass().getSimpleName()); - } - - @Override - public void visitWith(WithPattern pattern) - { - visitPrevious(pattern); - appendLine("with(%s)", pattern.getProperty().getName()); - level += 1; - pattern.getPattern().accept(this); - level -= 1; - } - - @Override - public void visitCapture(CapturePattern pattern) - { - visitPrevious(pattern); - appendLine("capturedAs(%s)", pattern.capture().description()); - } - - @Override - public void visitEquals(EqualsPattern pattern) - { - visitPrevious(pattern); - appendLine("equals(%s)", pattern.expectedValue()); - } - - @Override - public void visitFilter(FilterPattern pattern) - { - visitPrevious(pattern); - appendLine("filter(%s)", pattern.predicate()); - } - - @Override - public void visitOr(OrPattern pattern) - { - visitPrevious(pattern); - level += 1; - Iterator iterator = pattern.getPatterns().iterator(); - while (iterator.hasNext()) { - Pattern subPattern = (Pattern) iterator.next(); - subPattern.accept(this); - if (iterator.hasNext()) { - appendLine("or"); - } - } - level -= 1; - } - - private void appendLine(String template, Object... arguments) - { - result.append("\t".repeat(level)).append(format(template + "\n", arguments)); - } -} diff --git a/lib/trino-matching/src/main/java/io/trino/matching/pattern/EqualsPattern.java b/lib/trino-matching/src/main/java/io/trino/matching/EqualsPattern.java similarity index 79% rename from lib/trino-matching/src/main/java/io/trino/matching/pattern/EqualsPattern.java rename to lib/trino-matching/src/main/java/io/trino/matching/EqualsPattern.java index 3a6c18d52abd..4962b3105fdf 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/pattern/EqualsPattern.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/EqualsPattern.java @@ -11,19 +11,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.matching.pattern; - -import io.trino.matching.Captures; -import io.trino.matching.Match; -import io.trino.matching.Pattern; -import io.trino.matching.PatternVisitor; +package io.trino.matching; import java.util.Optional; import java.util.stream.Stream; import static java.util.Objects.requireNonNull; -public class EqualsPattern +public final class EqualsPattern extends Pattern { private final T expectedValue; @@ -45,10 +40,4 @@ public Stream accept(Object object, Captures captures, C context) return Stream.of(Match.of(captures)) .filter(match -> expectedValue.equals(object)); } - - @Override - public void accept(PatternVisitor patternVisitor) - { - patternVisitor.visitEquals(this); - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/pattern/FilterPattern.java b/lib/trino-matching/src/main/java/io/trino/matching/FilterPattern.java similarity index 81% rename from lib/trino-matching/src/main/java/io/trino/matching/pattern/FilterPattern.java rename to lib/trino-matching/src/main/java/io/trino/matching/FilterPattern.java index 0cea9186c364..454e200f9cbf 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/pattern/FilterPattern.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/FilterPattern.java @@ -11,12 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.matching.pattern; - -import io.trino.matching.Captures; -import io.trino.matching.Match; -import io.trino.matching.Pattern; -import io.trino.matching.PatternVisitor; +package io.trino.matching; import java.util.Optional; import java.util.function.BiPredicate; @@ -24,7 +19,7 @@ import static java.util.Objects.requireNonNull; -public class FilterPattern +public final class FilterPattern extends Pattern { private final BiPredicate predicate; @@ -48,10 +43,4 @@ public Stream accept(Object object, Captures captures, C context) return Stream.of(Match.of(captures)) .filter(match -> predicate.test((T) object, context)); } - - @Override - public void accept(PatternVisitor patternVisitor) - { - patternVisitor.visitFilter(this); - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/Match.java b/lib/trino-matching/src/main/java/io/trino/matching/Match.java index f7a6dfb631d4..d4710f1c2742 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/Match.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/Match.java @@ -13,59 +13,22 @@ */ package io.trino.matching; -import java.util.Objects; - -import static com.google.common.base.MoreObjects.toStringHelper; import static java.util.Objects.requireNonNull; -public final class Match +public record Match(Captures captures) { public static Match of(Captures captures) { return new Match(captures); } - private final Captures captures; - - private Match(Captures captures) + public Match { - this.captures = requireNonNull(captures, "captures is null"); + requireNonNull(captures, "captures is null"); } public T capture(Capture capture) { return captures().get(capture); } - - public Captures captures() - { - return captures; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Match match = (Match) o; - return Objects.equals(captures, match.captures); - } - - @Override - public int hashCode() - { - return Objects.hash(captures); - } - - @Override - public String toString() - { - return toStringHelper(this) - .add("captures", captures) - .toString(); - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/pattern/OrPattern.java b/lib/trino-matching/src/main/java/io/trino/matching/OrPattern.java similarity index 77% rename from lib/trino-matching/src/main/java/io/trino/matching/pattern/OrPattern.java rename to lib/trino-matching/src/main/java/io/trino/matching/OrPattern.java index 600f92d2d57e..01171e8e744b 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/pattern/OrPattern.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/OrPattern.java @@ -11,17 +11,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.matching.pattern; - -import io.trino.matching.Captures; -import io.trino.matching.Match; -import io.trino.matching.Pattern; -import io.trino.matching.PatternVisitor; +package io.trino.matching; import java.util.List; import java.util.stream.Stream; -public class OrPattern +public final class OrPattern extends Pattern { private final List> patterns; @@ -43,10 +38,4 @@ public Stream accept(Object object, Captures captures, C context) return patterns.stream() .flatMap(pattern -> pattern.accept(object, captures, context)); } - - @Override - public void accept(PatternVisitor patternVisitor) - { - patternVisitor.visitOr(this); - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/Pattern.java b/lib/trino-matching/src/main/java/io/trino/matching/Pattern.java index cbd6f7f29a8d..bbec51d63d2f 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/Pattern.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/Pattern.java @@ -14,11 +14,6 @@ package io.trino.matching; import com.google.common.collect.Iterables; -import io.trino.matching.pattern.CapturePattern; -import io.trino.matching.pattern.FilterPattern; -import io.trino.matching.pattern.OrPattern; -import io.trino.matching.pattern.TypeOfPattern; -import io.trino.matching.pattern.WithPattern; import java.util.Arrays; import java.util.List; @@ -32,7 +27,8 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Objects.requireNonNull; -public abstract class Pattern +public abstract sealed class Pattern + permits CapturePattern, CustomPattern, EqualsPattern, FilterPattern, OrPattern, TypeOfPattern, WithPattern { private final Optional> previous; @@ -105,8 +101,6 @@ public Optional> previous() public abstract Stream accept(Object object, Captures captures, C context); - public abstract void accept(PatternVisitor patternVisitor); - public boolean matches(Object object, C context) { return match(object, context) @@ -136,8 +130,8 @@ public final Stream match(Object object, Captures captures, C context @Override public String toString() { - DefaultPrinter printer = new DefaultPrinter(); - accept(printer); + PatternPrinter printer = new PatternPrinter(); + printer.print(this); return printer.result(); } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/PatternPrinter.java b/lib/trino-matching/src/main/java/io/trino/matching/PatternPrinter.java new file mode 100644 index 000000000000..58ed8a73a608 --- /dev/null +++ b/lib/trino-matching/src/main/java/io/trino/matching/PatternPrinter.java @@ -0,0 +1,64 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.matching; + +import java.util.Iterator; + +import static java.lang.String.format; + +public class PatternPrinter +{ + private final StringBuilder result = new StringBuilder(); + private int level; + + public String result() + { + return result.toString(); + } + + public void print(Pattern pattern) + { + pattern.previous().ifPresent(this::print); + switch (pattern) { + case TypeOfPattern typeOfPattern -> appendLine("typeOf(%s)", typeOfPattern.expectedClass().getSimpleName()); + case WithPattern withPattern -> { + appendLine("with(%s)", withPattern.getProperty().name()); + level += 1; + print(withPattern.getPattern()); + level -= 1; + } + case CapturePattern capturePattern -> appendLine("capturedAs(%s)", capturePattern.capture().description()); + case EqualsPattern equalsPattern -> appendLine("equals(%s)", equalsPattern.expectedValue()); + case FilterPattern filterPattern -> appendLine("filter(%s)", filterPattern.predicate()); + case OrPattern orPattern -> { + level += 1; + Iterator iterator = orPattern.getPatterns().iterator(); + while (iterator.hasNext()) { + Pattern subPattern = (Pattern) iterator.next(); + print(subPattern); + if (iterator.hasNext()) { + appendLine("or"); + } + } + level -= 1; + } + case CustomPattern customPattern -> appendLine(customPattern.print()); + } + } + + private void appendLine(String template, Object... arguments) + { + result.append("\t".repeat(level)).append(format(template + "\n", arguments)); + } +} diff --git a/lib/trino-matching/src/main/java/io/trino/matching/PatternVisitor.java b/lib/trino-matching/src/main/java/io/trino/matching/PatternVisitor.java deleted file mode 100644 index 4e809d5456d6..000000000000 --- a/lib/trino-matching/src/main/java/io/trino/matching/PatternVisitor.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.matching; - -import io.trino.matching.pattern.CapturePattern; -import io.trino.matching.pattern.EqualsPattern; -import io.trino.matching.pattern.FilterPattern; -import io.trino.matching.pattern.OrPattern; -import io.trino.matching.pattern.TypeOfPattern; -import io.trino.matching.pattern.WithPattern; - -import java.util.Optional; - -public interface PatternVisitor -{ - void visitTypeOf(TypeOfPattern pattern); - - void visitWith(WithPattern pattern); - - void visitCapture(CapturePattern pattern); - - void visitEquals(EqualsPattern equalsPattern); - - void visitFilter(FilterPattern pattern); - - void visitOr(OrPattern pattern); - - default void visitPrevious(Pattern pattern) - { - Optional> previous = pattern.previous(); - if (previous.isPresent()) { - previous.get().accept(this); - } - } -} diff --git a/lib/trino-matching/src/main/java/io/trino/matching/Property.java b/lib/trino-matching/src/main/java/io/trino/matching/Property.java index b565658f3316..660293de445a 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/Property.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/Property.java @@ -13,9 +13,6 @@ */ package io.trino.matching; -import io.trino.matching.pattern.EqualsPattern; -import io.trino.matching.pattern.FilterPattern; - import java.util.Optional; import java.util.function.BiFunction; import java.util.function.BiPredicate; @@ -24,14 +21,11 @@ import static java.util.Objects.requireNonNull; -public class Property +public record Property(String name, BiFunction> function) { - private final String name; - private final BiFunction> function; - public static Property property(String name, Function function) { - return property(name, (source, context) -> function.apply(source)); + return property(name, (source, _) -> function.apply(source)); } public static Property property(String name, BiFunction function) @@ -41,7 +35,7 @@ public static Property property(String name, BiFunction Property optionalProperty(String name, Function> function) { - return new Property<>(name, (source, context) -> function.apply(source)); + return new Property<>(name, (source, _) -> function.apply(source)); } public static Property optionalProperty(String name, BiFunction> function) @@ -49,15 +43,10 @@ public static Property optionalProperty(String name, BiFuncti return new Property<>(name, function); } - public Property(String name, BiFunction> function) - { - this.name = requireNonNull(name, "name is null"); - this.function = requireNonNull(function, "function is null"); - } - - public String getName() + public Property { - return name; + requireNonNull(name, "name is null"); + requireNonNull(function, "function is null"); } public BiFunction> getFunction() @@ -84,7 +73,7 @@ public PropertyPattern equalTo(T expectedValue) public PropertyPattern matching(Predicate predicate) { - return matching((t, context) -> predicate.test(t)); + return matching((t, _) -> predicate.test(t)); } public PropertyPattern matching(BiPredicate predicate) diff --git a/lib/trino-matching/src/main/java/io/trino/matching/pattern/TypeOfPattern.java b/lib/trino-matching/src/main/java/io/trino/matching/TypeOfPattern.java similarity index 80% rename from lib/trino-matching/src/main/java/io/trino/matching/pattern/TypeOfPattern.java rename to lib/trino-matching/src/main/java/io/trino/matching/TypeOfPattern.java index 5e2efc47abe8..cd85cef9c12f 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/pattern/TypeOfPattern.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/TypeOfPattern.java @@ -11,19 +11,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.matching.pattern; - -import io.trino.matching.Captures; -import io.trino.matching.Match; -import io.trino.matching.Pattern; -import io.trino.matching.PatternVisitor; +package io.trino.matching; import java.util.Optional; import java.util.stream.Stream; import static java.util.Objects.requireNonNull; -public class TypeOfPattern +public final class TypeOfPattern extends Pattern { private final Class expectedClass; @@ -52,10 +47,4 @@ public Stream accept(Object object, Captures captures, C context) } return Stream.of(); } - - @Override - public void accept(PatternVisitor patternVisitor) - { - patternVisitor.visitTypeOf(this); - } } diff --git a/lib/trino-matching/src/main/java/io/trino/matching/pattern/WithPattern.java b/lib/trino-matching/src/main/java/io/trino/matching/WithPattern.java similarity index 81% rename from lib/trino-matching/src/main/java/io/trino/matching/pattern/WithPattern.java rename to lib/trino-matching/src/main/java/io/trino/matching/WithPattern.java index 0eb8b771339e..c38bcea6755c 100644 --- a/lib/trino-matching/src/main/java/io/trino/matching/pattern/WithPattern.java +++ b/lib/trino-matching/src/main/java/io/trino/matching/WithPattern.java @@ -11,14 +11,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.matching.pattern; - -import io.trino.matching.Captures; -import io.trino.matching.Match; -import io.trino.matching.Pattern; -import io.trino.matching.PatternVisitor; -import io.trino.matching.Property; -import io.trino.matching.PropertyPattern; +package io.trino.matching; import java.util.Optional; import java.util.function.BiFunction; @@ -26,7 +19,7 @@ import static java.util.Objects.requireNonNull; -public class WithPattern +public final class WithPattern extends Pattern { private final PropertyPattern propertyPattern; @@ -56,10 +49,4 @@ public Stream accept(Object object, Captures captures, C context) return propertyValue.map(value -> propertyPattern.getPattern().match(value, captures, context)) .orElseGet(Stream::of); } - - @Override - public void accept(PatternVisitor patternVisitor) - { - patternVisitor.visitWith(this); - } } diff --git a/lib/trino-memory-context/pom.xml b/lib/trino-memory-context/pom.xml index 4946a4bf237f..459d5f7c48e5 100644 --- a/lib/trino-memory-context/pom.xml +++ b/lib/trino-memory-context/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-metastore/pom.xml b/lib/trino-metastore/pom.xml index 3ae2c759c2cb..fcf821ff23f3 100644 --- a/lib/trino-metastore/pom.xml +++ b/lib/trino-metastore/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -55,6 +55,11 @@ slice + + io.airlift + stats + + io.airlift units @@ -75,6 +80,11 @@ trino-cache + + io.trino + trino-plugin-toolkit + + io.trino trino-spi @@ -107,6 +117,13 @@ test + + io.trino + trino-spi + test-jar + test + + io.trino.hadoop hadoop-apache diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java index f1d0e95ce3c0..56e2e02701a4 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastore.java @@ -17,6 +17,7 @@ import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.function.LanguageFunction; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.RoleGrant; @@ -236,4 +237,9 @@ default void alterTransactionalTable(Table table, long transactionId, long write void replaceFunction(String databaseName, String functionName, LanguageFunction function); void dropFunction(String databaseName, String functionName, String signatureToken); + + default Metrics getMetrics() + { + return Metrics.EMPTY; + } } diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastoreFactory.java b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastoreFactory.java index 5b0564c8ffe0..69e7486be0ee 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastoreFactory.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/HiveMetastoreFactory.java @@ -35,25 +35,27 @@ default boolean hasBuiltInCaching() */ HiveMetastore createMetastore(Optional identity); - static HiveMetastoreFactory ofInstance(HiveMetastore metastore) + static HiveMetastoreFactory ofInstance(HiveMetastore metastore, boolean impersonationEnabled) { - return new StaticHiveMetastoreFactory(metastore); + return new StaticHiveMetastoreFactory(metastore, impersonationEnabled); } class StaticHiveMetastoreFactory implements HiveMetastoreFactory { private final HiveMetastore metastore; + private final boolean impersonationEnabled; - private StaticHiveMetastoreFactory(HiveMetastore metastore) + private StaticHiveMetastoreFactory(HiveMetastore metastore, boolean impersonationEnabled) { this.metastore = requireNonNull(metastore, "metastore is null"); + this.impersonationEnabled = impersonationEnabled; } @Override public boolean isImpersonationEnabled() { - return false; + return impersonationEnabled; } @Override diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/MeasuredHiveMetastore.java b/lib/trino-metastore/src/main/java/io/trino/metastore/MeasuredHiveMetastore.java new file mode 100644 index 000000000000..f748e7a26113 --- /dev/null +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/MeasuredHiveMetastore.java @@ -0,0 +1,507 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.metastore; + +import com.google.common.base.Throwables; +import com.google.common.base.Ticker; +import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.ThreadSafe; +import io.airlift.stats.TDigest; +import io.trino.plugin.base.metrics.DistributionSnapshot; +import io.trino.plugin.base.metrics.LongCount; +import io.trino.plugin.base.metrics.TDigestHistogram; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.function.LanguageFunction; +import io.trino.spi.metrics.Metric; +import io.trino.spi.metrics.Metrics; +import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.security.ConnectorIdentity; +import io.trino.spi.security.RoleGrant; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; + +import static java.util.Objects.requireNonNull; + +public class MeasuredHiveMetastore + implements HiveMetastore +{ + private final HiveMetastore delegate; + private final MetastoreApiCallStats allApiCallsStats = new MetastoreApiCallStats(); + private final Map apiCallStats = new ConcurrentHashMap<>(); + private final Ticker ticker = Ticker.systemTicker(); + + public MeasuredHiveMetastore(HiveMetastore delegate) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + } + + public static HiveMetastoreFactory factory(HiveMetastoreFactory metastoreFactory) + { + return new MeasuredMetastoreFactory(metastoreFactory); + } + + @Override + public Metrics getMetrics() + { + ImmutableMap.Builder> metrics = ImmutableMap.builder(); + allApiCallsStats.storeTo(metrics, "metastore.all"); + for (Map.Entry callStats : apiCallStats.entrySet()) { + callStats.getValue().storeTo(metrics, "metastore.%s".formatted(callStats.getKey())); + } + return new Metrics(metrics.buildOrThrow()); + } + + @Override + public Optional getDatabase(String databaseName) + { + return wrap("getDatabase", () -> delegate.getDatabase(databaseName)); + } + + @Override + public List getAllDatabases() + { + return wrap("getAllDatabases", delegate::getAllDatabases); + } + + @Override + public Optional getTable(String databaseName, String tableName) + { + return wrap("getTable", () -> delegate.getTable(databaseName, tableName)); + } + + @Override + public Map getTableColumnStatistics(String databaseName, String tableName, Set columnNames) + { + return wrap("getTableColumnStatistics", () -> delegate.getTableColumnStatistics(databaseName, tableName, columnNames)); + } + + @Override + public Map> getPartitionColumnStatistics(String databaseName, String tableName, Set partitionNames, Set columnNames) + { + return wrap("getPartitionColumnStatistics", () -> delegate.getPartitionColumnStatistics(databaseName, tableName, partitionNames, columnNames)); + } + + @Override + public boolean useSparkTableStatistics() + { + return wrap("useSparkTableStatistics", delegate::useSparkTableStatistics); + } + + @Override + public void updateTableStatistics(String databaseName, String tableName, OptionalLong acidWriteId, StatisticsUpdateMode mode, PartitionStatistics statisticsUpdate) + { + wrap("updateTableStatistics", () -> delegate.updateTableStatistics(databaseName, tableName, acidWriteId, mode, statisticsUpdate)); + } + + @Override + public void updatePartitionStatistics(Table table, StatisticsUpdateMode mode, Map partitionUpdates) + { + wrap("updatePartitionStatistics", () -> delegate.updatePartitionStatistics(table, mode, partitionUpdates)); + } + + @Override + public List getTables(String databaseName) + { + return wrap("getTables", () -> delegate.getTables(databaseName)); + } + + @Override + public List getTableNamesWithParameters(String databaseName, String parameterKey, Set parameterValues) + { + return wrap("getTableNamesWithParameters", () -> delegate.getTableNamesWithParameters(databaseName, parameterKey, parameterValues)); + } + + @Override + public void createDatabase(Database database) + { + wrap("createDatabase", () -> delegate.createDatabase(database)); + } + + @Override + public void dropDatabase(String databaseName, boolean deleteData) + { + wrap("dropDatabase", () -> delegate.dropDatabase(databaseName, deleteData)); + } + + @Override + public void renameDatabase(String databaseName, String newDatabaseName) + { + wrap("renameDatabase", () -> delegate.renameDatabase(databaseName, newDatabaseName)); + } + + @Override + public void setDatabaseOwner(String databaseName, HivePrincipal principal) + { + wrap("setDatabaseOwner", () -> delegate.setDatabaseOwner(databaseName, principal)); + } + + @Override + public void createTable(Table table, PrincipalPrivileges principalPrivileges) + { + wrap("createTable", () -> delegate.createTable(table, principalPrivileges)); + } + + @Override + public void dropTable(String databaseName, String tableName, boolean deleteData) + { + wrap("dropTable", () -> delegate.dropTable(databaseName, tableName, deleteData)); + } + + @Override + public void replaceTable(String databaseName, String tableName, Table newTable, PrincipalPrivileges principalPrivileges, Map environmentContext) + { + wrap("replaceTable", () -> delegate.replaceTable(databaseName, tableName, newTable, principalPrivileges, environmentContext)); + } + + @Override + public void renameTable(String databaseName, String tableName, String newDatabaseName, String newTableName) + { + wrap("renameTable", () -> delegate.renameTable(databaseName, tableName, newDatabaseName, newTableName)); + } + + @Override + public void commentTable(String databaseName, String tableName, Optional comment) + { + wrap("commentTable", () -> delegate.commentTable(databaseName, tableName, comment)); + } + + @Override + public void setTableOwner(String databaseName, String tableName, HivePrincipal principal) + { + wrap("setTableOwner", () -> delegate.setTableOwner(databaseName, tableName, principal)); + } + + @Override + public void commentColumn(String databaseName, String tableName, String columnName, Optional comment) + { + wrap("commentColumn", () -> delegate.commentColumn(databaseName, tableName, columnName, comment)); + } + + @Override + public void addColumn(String databaseName, String tableName, String columnName, HiveType columnType, String columnComment) + { + wrap("addColumn", () -> delegate.addColumn(databaseName, tableName, columnName, columnType, columnComment)); + } + + @Override + public void renameColumn(String databaseName, String tableName, String oldColumnName, String newColumnName) + { + wrap("renameColumn", () -> delegate.renameColumn(databaseName, tableName, oldColumnName, newColumnName)); + } + + @Override + public void dropColumn(String databaseName, String tableName, String columnName) + { + wrap("dropColumn", () -> delegate.dropColumn(databaseName, tableName, columnName)); + } + + @Override + public Optional getPartition(Table table, List partitionValues) + { + return wrap("getPartition", () -> delegate.getPartition(table, partitionValues)); + } + + @Override + public Optional> getPartitionNamesByFilter(String databaseName, String tableName, List columnNames, TupleDomain partitionKeysFilter) + { + return wrap("getPartitionNamesByFilter", () -> delegate.getPartitionNamesByFilter(databaseName, tableName, columnNames, partitionKeysFilter)); + } + + @Override + public Map> getPartitionsByNames(Table table, List partitionNames) + { + return wrap("getPartitionsByNames", () -> delegate.getPartitionsByNames(table, partitionNames)); + } + + @Override + public void addPartitions(String databaseName, String tableName, List partitions) + { + wrap("addPartitions", () -> delegate.addPartitions(databaseName, tableName, partitions)); + } + + @Override + public void dropPartition(String databaseName, String tableName, List parts, boolean deleteData) + { + wrap("dropPartition", () -> delegate.dropPartition(databaseName, tableName, parts, deleteData)); + } + + @Override + public void alterPartition(String databaseName, String tableName, PartitionWithStatistics partition) + { + wrap("alterPartition", () -> delegate.alterPartition(databaseName, tableName, partition)); + } + + @Override + public void createRole(String role, String grantor) + { + wrap("createRole", () -> delegate.createRole(role, grantor)); + } + + @Override + public void dropRole(String role) + { + wrap("dropRole", () -> delegate.dropRole(role)); + } + + @Override + public Set listRoles() + { + return wrap("listRoles", delegate::listRoles); + } + + @Override + public void grantRoles(Set roles, Set grantees, boolean adminOption, HivePrincipal grantor) + { + wrap("grantRoles", () -> delegate.grantRoles(roles, grantees, adminOption, grantor)); + } + + @Override + public void revokeRoles(Set roles, Set grantees, boolean adminOption, HivePrincipal grantor) + { + wrap("revokeRoles", () -> delegate.revokeRoles(roles, grantees, adminOption, grantor)); + } + + @Override + public Set listRoleGrants(HivePrincipal principal) + { + return wrap("listRoleGrants", () -> delegate.listRoleGrants(principal)); + } + + @Override + public void grantTablePrivileges(String databaseName, String tableName, String tableOwner, HivePrincipal grantee, HivePrincipal grantor, Set privileges, boolean grantOption) + { + wrap("grantTablePrivileges", () -> delegate.grantTablePrivileges(databaseName, tableName, tableOwner, grantee, grantor, privileges, grantOption)); + } + + @Override + public void revokeTablePrivileges(String databaseName, String tableName, String tableOwner, HivePrincipal grantee, HivePrincipal grantor, Set privileges, boolean grantOption) + { + wrap("revokeTablePrivileges", () -> delegate.revokeTablePrivileges(databaseName, tableName, tableOwner, grantee, grantor, privileges, grantOption)); + } + + @Override + public Set listTablePrivileges(String databaseName, String tableName, Optional tableOwner, Optional principal) + { + return wrap("listTablePrivileges", () -> delegate.listTablePrivileges(databaseName, tableName, tableOwner, principal)); + } + + @Override + public void checkSupportsTransactions() + { + wrap("checkSupportsTransactions", () -> delegate.checkSupportsTransactions()); + } + + @Override + public long openTransaction(AcidTransactionOwner transactionOwner) + { + return wrap("openTransaction", () -> delegate.openTransaction(transactionOwner)); + } + + @Override + public void commitTransaction(long transactionId) + { + wrap("commitTransaction", () -> delegate.commitTransaction(transactionId)); + } + + @Override + public void abortTransaction(long transactionId) + { + wrap("abortTransaction", () -> delegate.abortTransaction(transactionId)); + } + + @Override + public void sendTransactionHeartbeat(long transactionId) + { + wrap("sendTransactionHeartbeat", () -> delegate.sendTransactionHeartbeat(transactionId)); + } + + @Override + public void acquireSharedReadLock(AcidTransactionOwner transactionOwner, String queryId, long transactionId, List fullTables, List partitions) + { + wrap("acquireSharedReadLock", () -> delegate.acquireSharedReadLock(transactionOwner, queryId, transactionId, fullTables, partitions)); + } + + @Override + public String getValidWriteIds(List tables, long currentTransactionId) + { + return wrap("getValidWriteIds", () -> delegate.getValidWriteIds(tables, currentTransactionId)); + } + + @Override + public Optional getConfigValue(String name) + { + return wrap("getConfigValue", () -> delegate.getConfigValue(name)); + } + + @Override + public long allocateWriteId(String dbName, String tableName, long transactionId) + { + return wrap("allocateWriteId", () -> delegate.allocateWriteId(dbName, tableName, transactionId)); + } + + @Override + public void acquireTableWriteLock(AcidTransactionOwner transactionOwner, String queryId, long transactionId, String dbName, String tableName, AcidOperation operation, boolean isDynamicPartitionWrite) + { + wrap("acquireTableWriteLock", () -> delegate.acquireTableWriteLock(transactionOwner, queryId, transactionId, dbName, tableName, operation, isDynamicPartitionWrite)); + } + + @Override + public void updateTableWriteId(String dbName, String tableName, long transactionId, long writeId, OptionalLong rowCountChange) + { + wrap("updateTableWriteId", () -> delegate.updateTableWriteId(dbName, tableName, transactionId, writeId, rowCountChange)); + } + + @Override + public void addDynamicPartitions(String dbName, String tableName, List partitionNames, long transactionId, long writeId, AcidOperation operation) + { + wrap("addDynamicPartitions", () -> delegate.addDynamicPartitions(dbName, tableName, partitionNames, transactionId, writeId, operation)); + } + + @Override + public void alterTransactionalTable(Table table, long transactionId, long writeId, PrincipalPrivileges principalPrivileges) + { + wrap("alterTransactionalTable", () -> delegate.alterTransactionalTable(table, transactionId, writeId, principalPrivileges)); + } + + @Override + public boolean functionExists(String databaseName, String functionName, String signatureToken) + { + return wrap("functionExists", () -> delegate.functionExists(databaseName, functionName, signatureToken)); + } + + @Override + public Collection getAllFunctions(String databaseName) + { + return wrap("getAllFunctions", () -> delegate.getAllFunctions(databaseName)); + } + + @Override + public Collection getFunctions(String databaseName, String functionName) + { + return wrap("getFunctions", () -> delegate.getFunctions(databaseName, functionName)); + } + + @Override + public void createFunction(String databaseName, String functionName, LanguageFunction function) + { + wrap("createFunction", () -> delegate.createFunction(databaseName, functionName, function)); + } + + @Override + public void replaceFunction(String databaseName, String functionName, LanguageFunction function) + { + wrap("replaceFunction", () -> delegate.replaceFunction(databaseName, functionName, function)); + } + + @Override + public void dropFunction(String databaseName, String functionName, String signatureToken) + { + wrap("dropFunction", () -> delegate.dropFunction(databaseName, functionName, signatureToken)); + } + + private void wrap(String callName, Runnable call) + { + wrap(callName, () -> { + call.run(); + return null; + }); + } + + private T wrap(String callName, Callable call) + { + MetastoreApiCallStats callStats = apiCallStats.computeIfAbsent(callName, _ -> new MetastoreApiCallStats()); + long start = ticker.read(); + try { + return call.call(); + } + catch (Exception e) { + callStats.recordFailure(); + allApiCallsStats.recordFailure(); + Throwables.throwIfUnchecked(e); + throw new RuntimeException(e); + } + finally { + long timeNanos = ticker.read() - start; + callStats.record(timeNanos); + allApiCallsStats.record(timeNanos); + } + } + + @ThreadSafe + static class MetastoreApiCallStats + { + private final TDigest timeNanosDistribution = new TDigest(); + private long totalTimeNanos; + private long totalFailures; + + public synchronized void record(long nanos) + { + timeNanosDistribution.add(nanos); + totalTimeNanos += nanos; + } + + public synchronized void recordFailure() + { + totalFailures++; + } + + public synchronized void storeTo(ImmutableMap.Builder> metrics, String prefix) + { + // DistributionSnapshot does not retain reference to the histogram + metrics.put(prefix + ".time.distribution", DistributionSnapshot.fromDistribution(new TDigestHistogram(timeNanosDistribution))); + metrics.put(prefix + ".time.total", new LongCount(totalTimeNanos)); + + // do not add redundant 0 failures to make the metrics more concise + if (totalFailures > 0) { + metrics.put(prefix + ".failures", new LongCount(totalFailures)); + } + } + } + + public static class MeasuredMetastoreFactory + implements HiveMetastoreFactory + { + private final HiveMetastoreFactory metastoreFactory; + + public MeasuredMetastoreFactory(HiveMetastoreFactory metastoreFactory) + { + this.metastoreFactory = requireNonNull(metastoreFactory, "metastoreFactory is null"); + } + + @Override + public boolean isImpersonationEnabled() + { + return metastoreFactory.isImpersonationEnabled(); + } + + @Override + public boolean hasBuiltInCaching() + { + return metastoreFactory.hasBuiltInCaching(); + } + + @Override + public HiveMetastore createMetastore(Optional identity) + { + return new MeasuredHiveMetastore(metastoreFactory.createMetastore(identity)); + } + } +} diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastore.java b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastore.java index b19ed01277b9..2b3929530c15 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastore.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/cache/CachingHiveMetastore.java @@ -47,6 +47,7 @@ import io.trino.spi.TrinoException; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.function.LanguageFunction; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.RoleGrant; import org.weakref.jmx.Managed; @@ -1120,6 +1121,12 @@ public void dropFunction(String databaseName, String functionName, String signat delegate.dropFunction(databaseName, functionName, signatureToken); } + @Override + public Metrics getMetrics() + { + return delegate.getMetrics(); + } + private static LoadingCache buildCache( OptionalLong expiresAfterWriteMillis, OptionalLong refreshMillis, diff --git a/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java b/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java index 39c89d29eee9..3d5c109b1aaa 100644 --- a/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java +++ b/lib/trino-metastore/src/main/java/io/trino/metastore/tracing/TracingHiveMetastore.java @@ -33,6 +33,7 @@ import io.trino.metastore.TableInfo; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.function.LanguageFunction; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.RoleGrant; @@ -676,4 +677,10 @@ public void dropFunction(String databaseName, String functionName, String signat .startSpan(); withTracing(span, () -> delegate.dropFunction(databaseName, functionName, signatureToken)); } + + @Override + public Metrics getMetrics() + { + return delegate.getMetrics(); + } } diff --git a/lib/trino-metastore/src/test/java/io/trino/metastore/TestMeasuredHiveMetastore.java b/lib/trino-metastore/src/test/java/io/trino/metastore/TestMeasuredHiveMetastore.java new file mode 100644 index 000000000000..7bef542a6639 --- /dev/null +++ b/lib/trino-metastore/src/test/java/io/trino/metastore/TestMeasuredHiveMetastore.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.metastore; + +import io.trino.metastore.MeasuredHiveMetastore.MeasuredMetastoreFactory; +import org.junit.jupiter.api.Test; + +import static io.trino.spi.testing.InterfaceTestUtils.assertAllMethodsOverridden; + +class TestMeasuredHiveMetastore +{ + @Test + public void testAllMethodsImplemented() + { + assertAllMethodsOverridden(HiveMetastore.class, MeasuredHiveMetastore.class); + } + + @Test + public void testAllFactoryMethodsImplemented() + { + assertAllMethodsOverridden(HiveMetastoreFactory.class, MeasuredMetastoreFactory.class); + } +} diff --git a/lib/trino-orc/pom.xml b/lib/trino-orc/pom.xml index 4c708affa74a..a50b17ba4d0c 100644 --- a/lib/trino-orc/pom.xml +++ b/lib/trino-orc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-orc/src/main/java/io/trino/orc/DiskRange.java b/lib/trino-orc/src/main/java/io/trino/orc/DiskRange.java index 1717f47ec24d..c50b136edace 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/DiskRange.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/DiskRange.java @@ -84,7 +84,7 @@ public boolean equals(Object obj) } DiskRange other = (DiskRange) obj; return this.offset == other.offset && - this.length == other.length; + this.length == other.length; } @Override diff --git a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java index 0aa747d0eafb..e3e9948fee64 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/OrcWriter.java @@ -72,6 +72,7 @@ import static io.trino.orc.OrcWriterStats.FlushReason.DICTIONARY_FULL; import static io.trino.orc.OrcWriterStats.FlushReason.MAX_BYTES; import static io.trino.orc.OrcWriterStats.FlushReason.MAX_ROWS; +import static io.trino.orc.metadata.CalendarKind.PROLEPTIC_GREGORIAN; import static io.trino.orc.metadata.ColumnEncoding.ColumnEncodingKind.DIRECT; import static io.trino.orc.metadata.OrcColumnId.ROOT_COLUMN; import static io.trino.orc.metadata.PostScript.MAGIC; @@ -529,7 +530,8 @@ private List bufferFileFooter() orcTypes, fileStats, userMetadata, - Optional.empty()); // writer id will be set by MetadataWriter + Optional.empty(), // writer id will be set by MetadataWriter + PROLEPTIC_GREGORIAN); closedStripes.clear(); closedStripesRetainedBytes = 0; diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/CalendarKind.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/CalendarKind.java new file mode 100644 index 000000000000..468e241497f0 --- /dev/null +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/CalendarKind.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.orc.metadata; + +public enum CalendarKind +{ + UNKNOWN_CALENDAR, + JULIAN_GREGORIAN, + PROLEPTIC_GREGORIAN +} diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java index b1d2daf22ee3..2ba27f0cfbab 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/Footer.java @@ -37,6 +37,7 @@ public class Footer private final Optional> fileStats; private final Map userMetadata; private final Optional writerId; + private final CalendarKind calendar; public Footer( long numberOfRows, @@ -45,7 +46,8 @@ public Footer( ColumnMetadata types, Optional> fileStats, Map userMetadata, - Optional writerId) + Optional writerId, + CalendarKind calendar) { this.numberOfRows = numberOfRows; rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1")); @@ -56,6 +58,7 @@ public Footer( requireNonNull(userMetadata, "userMetadata is null"); this.userMetadata = ImmutableMap.copyOf(transformValues(userMetadata, Slice::copy)); this.writerId = requireNonNull(writerId, "writerId is null"); + this.calendar = requireNonNull(calendar, "calendar is null"); } public long getNumberOfRows() @@ -93,6 +96,11 @@ public Optional getWriterId() return writerId; } + public CalendarKind getCalendar() + { + return calendar; + } + @Override public String toString() { @@ -104,6 +112,7 @@ public String toString() .add("columnStatistics", fileStats) .add("userMetadata", userMetadata.keySet()) .add("writerId", writerId) + .add("calendar", calendar) .toString(); } } diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java index d32c36b90ae4..f0f5e4c4e50f 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataReader.java @@ -56,6 +56,9 @@ import static io.airlift.slice.SliceUtf8.lengthOfCodePoint; import static io.airlift.slice.SliceUtf8.tryGetCodePointAt; import static io.airlift.units.DataSize.Unit.GIGABYTE; +import static io.trino.orc.metadata.CalendarKind.JULIAN_GREGORIAN; +import static io.trino.orc.metadata.CalendarKind.PROLEPTIC_GREGORIAN; +import static io.trino.orc.metadata.CalendarKind.UNKNOWN_CALENDAR; import static io.trino.orc.metadata.CompressionKind.LZ4; import static io.trino.orc.metadata.CompressionKind.NONE; import static io.trino.orc.metadata.CompressionKind.SNAPPY; @@ -150,7 +153,8 @@ public Footer readFooter(HiveWriterVersion hiveWriterVersion, InputStream inputS toType(footer.getTypesList()), toColumnStatistics(hiveWriterVersion, footer.getStatisticsList(), false), toUserMetadata(footer.getMetadataList()), - Optional.of(footer.getWriter())); + Optional.of(footer.getWriter()), + toTrinoOrcCalendarKind(footer.getCalendar())); } private static List toStripeInformation(List types) @@ -409,6 +413,16 @@ private static BinaryStatistics toBinaryStatistics(OrcProto.BinaryStatistics bin return new BinaryStatistics(binaryStatistics.getSum()); } + private static CalendarKind toTrinoOrcCalendarKind(OrcProto.CalendarKind calendarKind) + { + return switch (calendarKind) { + case null -> UNKNOWN_CALENDAR; + case OrcProto.CalendarKind.UNKNOWN_CALENDAR -> UNKNOWN_CALENDAR; + case OrcProto.CalendarKind.JULIAN_GREGORIAN -> JULIAN_GREGORIAN; + case OrcProto.CalendarKind.PROLEPTIC_GREGORIAN -> PROLEPTIC_GREGORIAN; + }; + } + private static Slice byteStringToSlice(ByteString value) { return Slices.wrappedBuffer(value.toByteArray()); diff --git a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java index 5f62c3cc300e..d893e5c3bfde 100644 --- a/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java +++ b/lib/trino-orc/src/main/java/io/trino/orc/metadata/OrcMetadataWriter.java @@ -141,7 +141,8 @@ public int writeFooter(SliceOutput output, Footer footer) .collect(toList())) .addAllMetadata(footer.getUserMetadata().entrySet().stream() .map(OrcMetadataWriter::toUserMetadata) - .collect(toList())); + .collect(toList())) + .setCalendar(toOrcCalendarKind(footer.getCalendar())); setWriter(builder); @@ -361,6 +362,15 @@ private static OrcProto.Stream.Kind toStreamKind(StreamKind streamKind) throw new IllegalArgumentException("Unsupported stream kind: " + streamKind); } + private static OrcProto.CalendarKind toOrcCalendarKind(CalendarKind calendarKind) + { + return switch (calendarKind) { + case UNKNOWN_CALENDAR -> OrcProto.CalendarKind.UNKNOWN_CALENDAR; + case JULIAN_GREGORIAN -> OrcProto.CalendarKind.JULIAN_GREGORIAN; + case PROLEPTIC_GREGORIAN -> OrcProto.CalendarKind.PROLEPTIC_GREGORIAN; + }; + } + private static OrcProto.ColumnEncoding toColumnEncoding(ColumnEncoding columnEncodings) { return OrcProto.ColumnEncoding.newBuilder() diff --git a/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java b/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java index ce70f975d77c..56ce0e74aea2 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/OrcTester.java @@ -680,13 +680,13 @@ public static void writeOrcColumnTrino(File outputFile, CompressionKind compress } if (TIME_MICROS.equals(mappedType)) { return Optional.of(new OrcType( - LONG, - ImmutableList.of(), - ImmutableList.of(), - Optional.empty(), - Optional.empty(), - Optional.empty(), - ImmutableMap.of(ICEBERG_LONG_TYPE, "TIME"))); + LONG, + ImmutableList.of(), + ImmutableList.of(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + ImmutableMap.of(ICEBERG_LONG_TYPE, "TIME"))); } return Optional.empty(); })); diff --git a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java index 0cad039902a8..79fd5fb59822 100644 --- a/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java +++ b/lib/trino-orc/src/test/java/io/trino/orc/TestOrcWriter.java @@ -21,6 +21,7 @@ import io.airlift.units.DataSize; import io.trino.filesystem.local.LocalOutputFile; import io.trino.orc.OrcWriteValidation.OrcWriteValidationMode; +import io.trino.orc.metadata.CompressionKind; import io.trino.orc.metadata.Footer; import io.trino.orc.metadata.OrcMetadataReader; import io.trino.orc.metadata.OrcType; @@ -32,23 +33,33 @@ import io.trino.spi.Page; import io.trino.spi.block.Block; import io.trino.spi.block.VariableWidthBlockBuilder; +import io.trino.spi.type.SqlDate; +import io.trino.spi.type.SqlTimestamp; import io.trino.spi.type.Type; import org.junit.jupiter.api.Test; import java.io.IOException; import java.io.InputStream; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.ZoneId; import java.util.List; import java.util.Optional; +import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.units.DataSize.Unit.MEGABYTE; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.orc.OrcTester.READER_OPTIONS; import static io.trino.orc.StripeReader.isIndexStream; import static io.trino.orc.TestingOrcPredicate.ORC_ROW_GROUP_SIZE; import static io.trino.orc.TestingOrcPredicate.ORC_STRIPE_SIZE; +import static io.trino.orc.metadata.CalendarKind.PROLEPTIC_GREGORIAN; import static io.trino.orc.metadata.CompressionKind.NONE; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.testing.DateTimeTestingUtils.sqlDateOf; +import static io.trino.testing.DateTimeTestingUtils.sqlTimestampOf; import static java.lang.Math.toIntExact; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; @@ -78,6 +89,38 @@ public void testWriteHugeChunk() testWriteOutput(columnNameBuilder.build(), data); } + @Test + public void testCalendarEntryInFooter() + { + List strings = ImmutableList.of("aaa1", "qwerty", "asdf", "zxcvb", "1234"); + assertFooterHasProlepticGregorianCalendar(VARCHAR, strings); + + List dates = ImmutableList.of("2020-01-01", "2021-02-02", "2022-03-03", "2023-04-04", "2024-05-05").stream() + .map(text -> sqlDateOf(LocalDate.parse(text))) + .collect(toImmutableList()); + assertFooterHasProlepticGregorianCalendar(DATE, dates); + + List timestamps = ImmutableList.of("2023-04-11T05:16:12.123", "2021-04-11T05:16:12.123", "1999-04-11T05:16:12.123").stream() + .map(text -> sqlTimestampOf(TIMESTAMP_MILLIS.getPrecision(), LocalDateTime.parse(text))) + .collect(toImmutableList()); + assertFooterHasProlepticGregorianCalendar(TIMESTAMP_MILLIS, timestamps); + } + + private static void assertFooterHasProlepticGregorianCalendar(Type type, List values) + { + try (TempFile tempFile = new TempFile()) { + OrcTester.writeOrcColumnTrino(tempFile.getFile(), CompressionKind.NONE, type, values.iterator(), new OrcWriterStats()); + + OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), READER_OPTIONS); + + assertThat(OrcReader.createOrcReader(orcDataSource, READER_OPTIONS).orElseThrow(() -> new RuntimeException("File is empty")).getFooter().getCalendar()) + .isEqualTo(PROLEPTIC_GREGORIAN); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + private void testWriteOutput(List columnNames, String[] data) throws IOException { diff --git a/lib/trino-parquet/pom.xml b/lib/trino-parquet/pom.xml index 25db6136a6d5..0f56697dcff3 100644 --- a/lib/trino-parquet/pom.xml +++ b/lib/trino-parquet/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -101,6 +101,12 @@ + + io.trino + trino-filesystem + provided + + io.trino trino-spi @@ -119,12 +125,6 @@ runtime - - org.xerial.snappy - snappy-java - runtime - - io.airlift junit-extensions @@ -193,6 +193,12 @@ test + + org.apache.parquet + parquet-hadoop + test + + org.assertj assertj-core diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/BloomFilterStore.java b/lib/trino-parquet/src/main/java/io/trino/parquet/BloomFilterStore.java index 1afc665c1494..806625682082 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/BloomFilterStore.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/BloomFilterStore.java @@ -16,10 +16,15 @@ import com.google.common.collect.ImmutableMap; import io.airlift.slice.BasicSliceInput; import io.airlift.slice.Slice; +import io.trino.parquet.crypto.AesCipherUtils; +import io.trino.parquet.crypto.ColumnDecryptionContext; +import io.trino.parquet.crypto.FileDecryptionContext; +import io.trino.parquet.crypto.ModuleType; import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.TupleDomain; +import org.apache.parquet.bytes.BytesUtils; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.values.bloomfilter.BlockSplitBloomFilter; import org.apache.parquet.column.values.bloomfilter.BloomFilter; @@ -48,21 +53,27 @@ public class BloomFilterStore private final ParquetDataSource dataSource; private final Map bloomFilterOffsets; + private final Map columnChunks; + private final Optional decryptionContext; - public BloomFilterStore(ParquetDataSource dataSource, BlockMetadata block, Set columnsFiltered) + public BloomFilterStore(ParquetDataSource dataSource, BlockMetadata block, Set columnsFiltered, Optional decryptionContext) { this.dataSource = requireNonNull(dataSource, "dataSource is null"); requireNonNull(block, "block is null"); requireNonNull(columnsFiltered, "columnsFiltered is null"); + this.decryptionContext = requireNonNull(decryptionContext, "decryptionContext is null"); ImmutableMap.Builder bloomFilterOffsetBuilder = ImmutableMap.builder(); + ImmutableMap.Builder chunkBuilder = ImmutableMap.builder(); for (ColumnChunkMetadata column : block.columns()) { ColumnPath path = column.getPath(); if (hasBloomFilter(column) && columnsFiltered.contains(path)) { bloomFilterOffsetBuilder.put(path, column.getBloomFilterOffset()); + chunkBuilder.put(path, column); } } this.bloomFilterOffsets = bloomFilterOffsetBuilder.buildOrThrow(); + this.columnChunks = chunkBuilder.buildOrThrow(); } public Optional getBloomFilter(ColumnPath columnPath) @@ -74,9 +85,24 @@ public Optional getBloomFilter(ColumnPath columnPath) if (columnBloomFilterOffset == null) { return Optional.empty(); } - BasicSliceInput headerSliceInput = dataSource.readFully(columnBloomFilterOffset, MAX_HEADER_LENGTH).getInput(); - bloomFilterHeader = Util.readBloomFilterHeader(headerSliceInput); - bloomFilterDataOffset = columnBloomFilterOffset + headerSliceInput.position(); + // If the column is encrypted, decrypt the header using the metadata decryptor + Optional columnContext = decryptionContext.flatMap(context -> context.getColumnDecryptionContext(columnPath)); + if (columnContext.isPresent()) { + // Read encrypted header module: SIZE(4) + NONCE + CIPHERTEXT + TAG + int encryptedSize = BytesUtils.readIntLittleEndian(dataSource.readFully(columnBloomFilterOffset, AesCipherUtils.SIZE_LENGTH).getBytes(), 0); + Slice module = dataSource.readFully(columnBloomFilterOffset, AesCipherUtils.SIZE_LENGTH + encryptedSize); + BasicSliceInput in = module.getInput(); + ColumnChunkMetadata chunk = requireNonNull(columnChunks.get(columnPath), "missing chunk metadata"); + byte[] aad = AesCipherUtils.createModuleAAD(columnContext.get().fileAad(), ModuleType.BloomFilterHeader, chunk.getRowGroupOrdinal(), chunk.getColumnOrdinal(), -1); + bloomFilterHeader = Util.readBloomFilterHeader(in, columnContext.get().metadataDecryptor(), aad); + // after read, position() == 4 + encrypted data length + bloomFilterDataOffset = columnBloomFilterOffset + in.position(); + } + else { + BasicSliceInput headerSliceInput = dataSource.readFully(columnBloomFilterOffset, MAX_HEADER_LENGTH).getInput(); + bloomFilterHeader = Util.readBloomFilterHeader(headerSliceInput); + bloomFilterDataOffset = columnBloomFilterOffset + headerSliceInput.position(); + } } catch (IOException exception) { throw new UncheckedIOException("Failed to read Bloom filter header", exception); @@ -87,9 +113,23 @@ public Optional getBloomFilter(ColumnPath columnPath) } try { - Slice bloomFilterData = dataSource.readFully(bloomFilterDataOffset, bloomFilterHeader.getNumBytes()); - verify(bloomFilterData.length() > 0, "Read empty bloom filter %s", bloomFilterHeader); - return Optional.of(new BlockSplitBloomFilter(bloomFilterData.getBytes())); + Optional columnContext = decryptionContext.flatMap(context -> context.getColumnDecryptionContext(columnPath)); + if (columnContext.isPresent()) { + // Read the whole bitset module: SIZE + NONCE + CIPHERTEXT + TAG + int encryptedSize = BytesUtils.readIntLittleEndian(dataSource.readFully(bloomFilterDataOffset, AesCipherUtils.SIZE_LENGTH).getBytes(), 0); + Slice module = dataSource.readFully(bloomFilterDataOffset, AesCipherUtils.SIZE_LENGTH + encryptedSize); + ColumnChunkMetadata chunk = requireNonNull(columnChunks.get(columnPath), "missing chunk metadata"); + byte[] aad = AesCipherUtils.createModuleAAD(columnContext.get().fileAad(), ModuleType.BloomFilterBitset, chunk.getRowGroupOrdinal(), chunk.getColumnOrdinal(), -1); + byte[] plain = columnContext.get().metadataDecryptor().decrypt(module.getBytes(), aad); + verify(plain.length == bloomFilterHeader.getNumBytes(), "Decrypted bloom filter length mismatch: expected %s, got %s", bloomFilterHeader.getNumBytes(), plain.length); + return Optional.of(new BlockSplitBloomFilter(plain)); + } + else { + // Plaintext bitset + Slice bloomFilterData = dataSource.readFully(bloomFilterDataOffset, bloomFilterHeader.getNumBytes()); + verify(bloomFilterData.length() > 0, "Read empty bloom filter %s", bloomFilterHeader); + return Optional.of(new BlockSplitBloomFilter(bloomFilterData.getBytes())); + } } catch (IOException exception) { throw new UncheckedIOException("Failed to read Bloom filter data", exception); @@ -100,7 +140,8 @@ public static Optional getBloomFilterStore( ParquetDataSource dataSource, BlockMetadata blockMetadata, TupleDomain parquetTupleDomain, - ParquetReaderOptions options) + ParquetReaderOptions options, + Optional decryptionContext) { if (!options.useBloomFilter() || parquetTupleDomain.isAll() || parquetTupleDomain.isNone()) { return Optional.empty(); @@ -117,7 +158,7 @@ public static Optional getBloomFilterStore( .map(column -> ColumnPath.get(column.getPath())) .collect(toImmutableSet()); - return Optional.of(new BloomFilterStore(dataSource, blockMetadata, columnsFilteredPaths)); + return Optional.of(new BloomFilterStore(dataSource, blockMetadata, columnsFilteredPaths, decryptionContext)); } public static boolean hasBloomFilter(ColumnChunkMetadata columnMetaData) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/ChunkKey.java b/lib/trino-parquet/src/main/java/io/trino/parquet/ChunkKey.java index 473ec7ee6a0b..597dd398fbb0 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/ChunkKey.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/ChunkKey.java @@ -45,7 +45,7 @@ public boolean equals(Object obj) } ChunkKey other = (ChunkKey) obj; return this.column == other.column && - this.rowGroup == other.rowGroup; + this.rowGroup == other.rowGroup; } @Override diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java index bbece17c9b7e..8eaebc60c93d 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPage.java @@ -21,12 +21,14 @@ public abstract sealed class DataPage { protected final int valueCount; private final OptionalLong firstRowIndex; + private final int pageIndex; - public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex) + public DataPage(int uncompressedSize, int valueCount, OptionalLong firstRowIndex, int pageIndex) { super(uncompressedSize); this.valueCount = valueCount; this.firstRowIndex = firstRowIndex; + this.pageIndex = pageIndex; } /** @@ -41,4 +43,9 @@ public int getValueCount() { return valueCount; } + + public int getPageIndex() + { + return pageIndex; + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java index b0895445d813..8dbf9809378d 100755 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV1.java @@ -35,15 +35,17 @@ public DataPageV1( OptionalLong firstRowIndex, ParquetEncoding repetitionLevelEncoding, ParquetEncoding definitionLevelEncoding, - ParquetEncoding valuesEncoding) + ParquetEncoding valuesEncoding, + int pageIndex) { - super(uncompressedSize, valueCount, firstRowIndex); + super(uncompressedSize, valueCount, firstRowIndex, pageIndex); this.slice = requireNonNull(slice, "slice is null"); this.repetitionLevelEncoding = repetitionLevelEncoding; this.definitionLevelEncoding = definitionLevelEncoding; this.valuesEncoding = valuesEncoding; } + @Override public Slice getSlice() { return slice; diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java index b0cbfd9ed8fc..6544942e74eb 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/DataPageV2.java @@ -44,9 +44,10 @@ public DataPageV2( int uncompressedSize, OptionalLong firstRowIndex, Statistics statistics, - boolean isCompressed) + boolean isCompressed, + int pageIndex) { - super(uncompressedSize, valueCount, firstRowIndex); + super(uncompressedSize, valueCount, firstRowIndex, pageIndex); this.rowCount = rowCount; this.nullCount = nullCount; this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null"); @@ -82,6 +83,7 @@ public ParquetEncoding getDataEncoding() return dataEncoding; } + @Override public Slice getSlice() { return slice; diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java index 74fdf540199d..bd92d7fc0c8e 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/DictionaryPage.java @@ -43,6 +43,7 @@ public DictionaryPage(Slice slice, int uncompressedSize, int dictionarySize, Par encoding); } + @Override public Slice getSlice() { return slice; diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/DiskRange.java b/lib/trino-parquet/src/main/java/io/trino/parquet/DiskRange.java index 14ad2fcb7adf..d0bebae36865 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/DiskRange.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/DiskRange.java @@ -84,7 +84,7 @@ public boolean equals(Object obj) } DiskRange other = (DiskRange) obj; return this.offset == other.offset && - this.length == other.length; + this.length == other.length; } @Override diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/Page.java b/lib/trino-parquet/src/main/java/io/trino/parquet/Page.java index 69cde62cf435..64b1f861717b 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/Page.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/Page.java @@ -13,6 +13,8 @@ */ package io.trino.parquet; +import io.airlift.slice.Slice; + public abstract class Page { protected final int uncompressedSize; @@ -26,4 +28,6 @@ public int getUncompressedSize() { return uncompressedSize; } + + public abstract Slice getSlice(); } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetValidationUtils.java b/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetValidationUtils.java index 5e31a08f704e..5a1d06861f60 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetValidationUtils.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/ParquetValidationUtils.java @@ -14,6 +14,7 @@ package io.trino.parquet; import com.google.errorprone.annotations.FormatMethod; +import io.trino.parquet.crypto.ParquetCryptoException; public final class ParquetValidationUtils { @@ -27,4 +28,12 @@ public static void validateParquet(boolean condition, ParquetDataSourceId dataSo throw new ParquetCorruptionException(dataSourceId, formatString, args); } } + + @FormatMethod + public static void validateParquetCrypto(boolean condition, ParquetDataSourceId dataSourceId, String formatString, Object... args) + { + if (!condition) { + throw new ParquetCryptoException(dataSourceId, formatString, args); + } + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesCipherUtils.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesCipherUtils.java new file mode 100644 index 000000000000..25f872daa68f --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesCipherUtils.java @@ -0,0 +1,150 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import java.io.IOException; +import java.io.InputStream; + +import static com.google.common.primitives.Bytes.concat; +import static java.util.Objects.requireNonNull; + +public final class AesCipherUtils +{ + public static final int SIZE_LENGTH = 4; + public static final int NONCE_LENGTH = 12; + public static final int GCM_TAG_LENGTH = 16; + public static final int CTR_IV_LENGTH = 16; + public static final int GCM_TAG_LENGTH_BITS = 8 * GCM_TAG_LENGTH; + public static final int CHUNK_LENGTH = 4 * 1024; + // NIST SP 800-38D section 8.3 specifies limit on AES GCM encryption operations with same key and random IV/nonce + public static final long GCM_RANDOM_IV_SAME_KEY_MAX_OPS = 1L << 32; + + private AesCipherUtils() {} + + public static void validateKeyBytes(byte[] keyBytes) + { + requireNonNull(keyBytes, "key bytes cannot be null"); + boolean allZeroKey = true; + for (byte kb : keyBytes) { + if (kb != 0) { + allZeroKey = false; + break; + } + } + + if (allZeroKey) { + throw new IllegalArgumentException("All key bytes are zero"); + } + } + + public static byte[] createModuleAAD(byte[] fileAAD, ModuleType moduleType, int rowGroupOrdinal, int columnOrdinal, int pageOrdinal) + { + byte[] typeOrdinalBytes = new byte[1]; + typeOrdinalBytes[0] = moduleType.getValue(); + + if (ModuleType.Footer == moduleType) { + return concat(fileAAD, typeOrdinalBytes); + } + + if (rowGroupOrdinal < 0) { + throw new IllegalArgumentException("Wrong row group ordinal: " + rowGroupOrdinal); + } + short shortRGOrdinal = (short) rowGroupOrdinal; + if (shortRGOrdinal != rowGroupOrdinal) { + throw new ParquetCryptoException("Encrypted parquet files can't have more than %s row groups: %s", Short.MAX_VALUE, rowGroupOrdinal); + } + byte[] rowGroupOrdinalBytes = shortToBytesLittleEndian(shortRGOrdinal); + + if (columnOrdinal < 0) { + throw new IllegalArgumentException("Wrong column ordinal: " + columnOrdinal); + } + short shortColumOrdinal = (short) columnOrdinal; + if (shortColumOrdinal != columnOrdinal) { + throw new ParquetCryptoException("Encrypted parquet files can't have more than %s columns: %s", Short.MAX_VALUE, columnOrdinal); + } + byte[] columnOrdinalBytes = shortToBytesLittleEndian(shortColumOrdinal); + + if (ModuleType.DataPage != moduleType && ModuleType.DataPageHeader != moduleType) { + return concat(fileAAD, typeOrdinalBytes, rowGroupOrdinalBytes, columnOrdinalBytes); + } + + if (pageOrdinal < 0) { + throw new IllegalArgumentException("Wrong page ordinal: " + pageOrdinal); + } + short shortPageOrdinal = (short) pageOrdinal; + if (shortPageOrdinal != pageOrdinal) { + throw new ParquetCryptoException("Encrypted parquet files can't have more than %s pages per chunk: %s", Short.MAX_VALUE, pageOrdinal); + } + byte[] pageOrdinalBytes = shortToBytesLittleEndian(shortPageOrdinal); + + return concat(fileAAD, typeOrdinalBytes, rowGroupOrdinalBytes, columnOrdinalBytes, pageOrdinalBytes); + } + + public static byte[] createFooterAAD(byte[] aadPrefixBytes) + { + return createModuleAAD(aadPrefixBytes, ModuleType.Footer, -1, -1, -1); + } + + // Update last two bytes with new page ordinal (instead of creating new page AAD from scratch) + public static void quickUpdatePageAAD(byte[] pageAAD, int newPageOrdinal) + { + requireNonNull(pageAAD, "pageAAD cannot be null"); + if (newPageOrdinal < 0) { + throw new IllegalArgumentException("Wrong page ordinal: " + newPageOrdinal); + } + short shortPageOrdinal = (short) newPageOrdinal; + if (shortPageOrdinal != newPageOrdinal) { + throw new ParquetCryptoException("Encrypted parquet files can't have more than %s pages per chunk: %s", Short.MAX_VALUE, newPageOrdinal); + } + + byte[] pageOrdinalBytes = shortToBytesLittleEndian(shortPageOrdinal); + System.arraycopy(pageOrdinalBytes, 0, pageAAD, pageAAD.length - 2, 2); + } + + public static int readCiphertextLength(InputStream from) + throws IOException + { + byte[] lengthBuffer = new byte[SIZE_LENGTH]; + int readBytes = 0; + + // Read the length of encrypted Thrift structure + while (readBytes < SIZE_LENGTH) { + int n = from.read(lengthBuffer, readBytes, SIZE_LENGTH - readBytes); + if (n <= 0) { + throw new IOException("Tried to read int (4 bytes), but only got " + readBytes + " bytes."); + } + readBytes += n; + } + + int ciphertextLength = ((lengthBuffer[3] & 0xff) << 24) + | ((lengthBuffer[2] & 0xff) << 16) + | ((lengthBuffer[1] & 0xff) << 8) + | (lengthBuffer[0] & 0xff); + + if (ciphertextLength < 1) { + throw new IOException("Wrong length of encrypted metadata: " + ciphertextLength); + } + + return ciphertextLength; + } + + private static byte[] shortToBytesLittleEndian(short input) + { + byte[] output = new byte[2]; + output[1] = (byte) (0xff & (input >> 8)); + output[0] = (byte) (0xff & input); + + return output; + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesCtrDecryptor.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesCtrDecryptor.java new file mode 100644 index 000000000000..b4740c63ccae --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesCtrDecryptor.java @@ -0,0 +1,186 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import org.apache.parquet.format.BlockCipher; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.Objects; + +import static io.trino.parquet.crypto.AesCipherUtils.CHUNK_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.CTR_IV_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.NONCE_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.SIZE_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.readCiphertextLength; +import static io.trino.parquet.crypto.AesCipherUtils.validateKeyBytes; + +public class AesCtrDecryptor + implements BlockCipher.Decryptor +{ + private final byte[] keyBytes; + private final Cipher cipher; + private final SecretKeySpec aesKey; + private final byte[] ctrIV; + + public AesCtrDecryptor(byte[] keyBytes) + { + validateKeyBytes(keyBytes); + this.keyBytes = keyBytes; + + try { + cipher = Cipher.getInstance(AesMode.CTR.getCipherName()); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to create CTR cipher"); + } + aesKey = new SecretKeySpec(keyBytes, "AES"); + ctrIV = new byte[CTR_IV_LENGTH]; + // Setting last bit of initial CTR counter to 1 + ctrIV[CTR_IV_LENGTH - 1] = (byte) 1; + } + + @Override + public byte[] decrypt(byte[] lengthAndCiphertext, byte[] aad) + { + return decrypt(lengthAndCiphertext, SIZE_LENGTH, lengthAndCiphertext.length - SIZE_LENGTH, aad); + } + + public byte[] decrypt(byte[] ciphertext, int cipherTextOffset, int cipherTextLength, byte[] aad) + throws ParquetCryptoException + { + int plainTextLength = cipherTextLength - NONCE_LENGTH; + if (plainTextLength < 1) { + throw new ParquetCryptoException("Wrong input length %s", plainTextLength); + } + + // Get the nonce from ciphertext + System.arraycopy(ciphertext, cipherTextOffset, ctrIV, 0, NONCE_LENGTH); + + byte[] plainText = new byte[plainTextLength]; + int inputLength = cipherTextLength - NONCE_LENGTH; + int inputOffset = cipherTextOffset + NONCE_LENGTH; + int outputOffset = 0; + try { + IvParameterSpec spec = new IvParameterSpec(ctrIV); + cipher.init(Cipher.DECRYPT_MODE, aesKey, spec); + + // Breaking decryption into multiple updates, to trigger h/w acceleration in Java 9+ + while (inputLength > CHUNK_LENGTH) { + int written = cipher.update(ciphertext, inputOffset, CHUNK_LENGTH, plainText, outputOffset); + inputOffset += CHUNK_LENGTH; + outputOffset += written; + inputLength -= CHUNK_LENGTH; + } + + cipher.doFinal(ciphertext, inputOffset, inputLength, plainText, outputOffset); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to decrypt"); + } + + return plainText; + } + + @Override + public ByteBuffer decrypt(ByteBuffer ciphertext, byte[] aad) + { + int cipherTextOffset = SIZE_LENGTH; + int cipherTextLength = ciphertext.limit() - ciphertext.position() - SIZE_LENGTH; + + int plainTextLength = cipherTextLength - NONCE_LENGTH; + if (plainTextLength < 1) { + throw new ParquetCryptoException("Wrong input length %s", plainTextLength); + } + + // skip size + ciphertext.position(ciphertext.position() + cipherTextOffset); + // Get the nonce from ciphertext + ciphertext.get(ctrIV, 0, NONCE_LENGTH); + + // Reuse the input buffer as the output buffer + ByteBuffer plainText = ciphertext.slice(); + plainText.limit(plainTextLength); + int inputLength = cipherTextLength - NONCE_LENGTH; + int inputOffset = cipherTextOffset + NONCE_LENGTH; + try { + IvParameterSpec spec = new IvParameterSpec(ctrIV); + cipher.init(Cipher.DECRYPT_MODE, aesKey, spec); + + // Breaking decryption into multiple updates, to trigger h/w acceleration in Java 9+ + while (inputLength > CHUNK_LENGTH) { + ciphertext.position(inputOffset); + ciphertext.limit(inputOffset + CHUNK_LENGTH); + cipher.update(ciphertext, plainText); + inputOffset += CHUNK_LENGTH; + inputLength -= CHUNK_LENGTH; + } + ciphertext.position(inputOffset); + ciphertext.limit(inputOffset + inputLength); + cipher.doFinal(ciphertext, plainText); + plainText.flip(); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to decrypt"); + } + + return plainText; + } + + @Override + public byte[] decrypt(InputStream from, byte[] aad) + throws IOException + { + int ciphertextLength = readCiphertextLength(from); + // Read the encrypted structure contents + byte[] ciphertextBuffer = new byte[ciphertextLength]; + int readBytes = 0; + while (readBytes < ciphertextLength) { + int n = from.read(ciphertextBuffer, readBytes, ciphertextLength - readBytes); + if (n <= 0) { + throw new IOException( + "Tried to read " + ciphertextLength + " bytes, but only got " + readBytes + " bytes."); + } + readBytes += n; + } + + // Decrypt the structure contents + return decrypt(ciphertextBuffer, 0, ciphertextLength, aad); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof AesCtrDecryptor that)) { + return false; + } + return Objects.deepEquals(keyBytes, that.keyBytes); + } + + @Override + public int hashCode() + { + return Arrays.hashCode(keyBytes); + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesGcmDecryptor.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesGcmDecryptor.java new file mode 100644 index 000000000000..d1b716be5cd1 --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesGcmDecryptor.java @@ -0,0 +1,175 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import org.apache.parquet.format.BlockCipher; + +import javax.crypto.AEADBadTagException; +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.security.GeneralSecurityException; +import java.util.Arrays; +import java.util.Objects; + +import static io.trino.parquet.crypto.AesCipherUtils.GCM_TAG_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.GCM_TAG_LENGTH_BITS; +import static io.trino.parquet.crypto.AesCipherUtils.NONCE_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.SIZE_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.readCiphertextLength; +import static io.trino.parquet.crypto.AesCipherUtils.validateKeyBytes; + +public class AesGcmDecryptor + implements BlockCipher.Decryptor +{ + private final byte[] keyBytes; + private final Cipher cipher; + private final SecretKeySpec aesKey; + private final byte[] localNonce; + + public AesGcmDecryptor(byte[] keyBytes) + { + validateKeyBytes(keyBytes); + this.keyBytes = keyBytes; + + try { + cipher = Cipher.getInstance(AesMode.GCM.getCipherName()); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to create GCM cipher"); + } + + aesKey = new SecretKeySpec(keyBytes, "AES"); + localNonce = new byte[NONCE_LENGTH]; + } + + @Override + public byte[] decrypt(byte[] lengthAndCiphertext, byte[] aad) + { + return decrypt(lengthAndCiphertext, SIZE_LENGTH, lengthAndCiphertext.length - SIZE_LENGTH, aad); + } + + public byte[] decrypt(byte[] ciphertext, int cipherTextOffset, int cipherTextLength, byte[] aad) + { + int plainTextLength = cipherTextLength - GCM_TAG_LENGTH - NONCE_LENGTH; + if (plainTextLength < 1) { + throw new ParquetCryptoException("Wrong input length %s", plainTextLength); + } + + // Get the nonce from ciphertext + System.arraycopy(ciphertext, cipherTextOffset, localNonce, 0, NONCE_LENGTH); + + byte[] plainText = new byte[plainTextLength]; + int inputLength = cipherTextLength - NONCE_LENGTH; + int inputOffset = cipherTextOffset + NONCE_LENGTH; + int outputOffset = 0; + try { + GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH_BITS, localNonce); + cipher.init(Cipher.DECRYPT_MODE, aesKey, spec); + if (null != aad) { + cipher.updateAAD(aad); + } + + cipher.doFinal(ciphertext, inputOffset, inputLength, plainText, outputOffset); + } + catch (AEADBadTagException e) { + throw new ParquetCryptoException(e, "GCM tag check failed"); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to decrypt"); + } + + return plainText; + } + + @Override + public ByteBuffer decrypt(ByteBuffer ciphertext, byte[] aad) + { + int cipherTextOffset = SIZE_LENGTH; + int cipherTextLength = ciphertext.limit() - ciphertext.position() - SIZE_LENGTH; + int plainTextLength = cipherTextLength - GCM_TAG_LENGTH - NONCE_LENGTH; + if (plainTextLength < 1) { + throw new ParquetCryptoException("Wrong input length %s", plainTextLength); + } + + ciphertext.position(ciphertext.position() + cipherTextOffset); + // Get the nonce from ciphertext + ciphertext.get(localNonce); + + // Reuse the input buffer as the output buffer + ByteBuffer plainText = ciphertext.slice(); + plainText.limit(plainTextLength); + try { + GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH_BITS, localNonce); + cipher.init(Cipher.DECRYPT_MODE, aesKey, spec); + if (null != aad) { + cipher.updateAAD(aad); + } + + cipher.doFinal(ciphertext, plainText); + plainText.flip(); + } + catch (AEADBadTagException e) { + throw new ParquetCryptoException(e, "GCM tag check failed"); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to decrypt"); + } + + return plainText; + } + + @Override + public byte[] decrypt(InputStream from, byte[] aad) + throws IOException + { + int ciphertextLength = readCiphertextLength(from); + // Read the encrypted structure contents + byte[] ciphertextBuffer = new byte[ciphertextLength]; + int readBytes = 0; + // Read the encrypted structure contents + while (readBytes < ciphertextLength) { + int n = from.read(ciphertextBuffer, readBytes, ciphertextLength - readBytes); + if (n <= 0) { + throw new IOException("Tried to read " + ciphertextLength + " bytes, but only got " + readBytes + " bytes."); + } + readBytes += n; + } + + // Decrypt the structure contents + return decrypt(ciphertextBuffer, 0, ciphertextLength, aad); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof AesGcmDecryptor that)) { + return false; + } + return Objects.deepEquals(keyBytes, that.keyBytes); + } + + @Override + public int hashCode() + { + return Arrays.hashCode(keyBytes); + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesGcmEncryptor.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesGcmEncryptor.java new file mode 100644 index 000000000000..f2034731214d --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesGcmEncryptor.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.format.BlockCipher; + +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import java.security.GeneralSecurityException; +import java.security.SecureRandom; + +import static io.trino.parquet.crypto.AesCipherUtils.GCM_RANDOM_IV_SAME_KEY_MAX_OPS; +import static io.trino.parquet.crypto.AesCipherUtils.GCM_TAG_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.GCM_TAG_LENGTH_BITS; +import static io.trino.parquet.crypto.AesCipherUtils.NONCE_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.SIZE_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.validateKeyBytes; + +public class AesGcmEncryptor + implements BlockCipher.Encryptor +{ + protected Cipher cipher; + protected final SecureRandom randomGenerator; + protected final byte[] localNonce; + protected SecretKeySpec aesKey; + + private long operationCounter; + + public AesGcmEncryptor(byte[] keyBytes) + { + validateKeyBytes(keyBytes); + operationCounter = 0; + + try { + cipher = Cipher.getInstance(AesMode.GCM.getCipherName()); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to create GCM cipher"); + } + + aesKey = new SecretKeySpec(keyBytes, "AES"); + randomGenerator = new SecureRandom(); + localNonce = new byte[NONCE_LENGTH]; + } + + @Override + public byte[] encrypt(byte[] plainText, byte[] aad) + { + return encrypt(true, plainText, aad); + } + + public byte[] encrypt(boolean writeLength, byte[] plainText, byte[] aad) + { + randomGenerator.nextBytes(localNonce); + return encrypt(writeLength, plainText, localNonce, aad); + } + + public byte[] encrypt(boolean writeLength, byte[] plainText, byte[] nonce, byte[] aad) + { + if (operationCounter > GCM_RANDOM_IV_SAME_KEY_MAX_OPS) { + throw new ParquetCryptoException("Exceeded limit of AES GCM encryption operations with same key and random IV"); + } + operationCounter++; + + if (nonce.length != NONCE_LENGTH) { + throw new ParquetCryptoException("Wrong nonce length %s", nonce.length); + } + int plainTextLength = plainText.length; + int cipherTextLength = NONCE_LENGTH + plainTextLength + GCM_TAG_LENGTH; + int lengthBufferLength = writeLength ? SIZE_LENGTH : 0; + byte[] cipherText = new byte[lengthBufferLength + cipherTextLength]; + int inputOffset = 0; + int outputOffset = lengthBufferLength + NONCE_LENGTH; + + try { + GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH_BITS, nonce); + cipher.init(Cipher.ENCRYPT_MODE, aesKey, spec); + if (null != aad) { + cipher.updateAAD(aad); + } + + cipher.doFinal(plainText, inputOffset, plainTextLength, cipherText, outputOffset); + } + catch (GeneralSecurityException e) { + throw new ParquetCryptoException(e, "Failed to encrypt"); + } + + // Add ciphertext length + if (writeLength) { + System.arraycopy(BytesUtils.intToBytes(cipherTextLength), 0, cipherText, 0, lengthBufferLength); + } + // Add the nonce + System.arraycopy(nonce, 0, cipherText, lengthBufferLength, NONCE_LENGTH); + + return cipherText; + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesMode.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesMode.java new file mode 100644 index 000000000000..e8affac6c9f0 --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/AesMode.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +public enum AesMode +{ + GCM("AES/GCM/NoPadding"), + CTR("AES/CTR/NoPadding"); + + private final String cipherName; + + AesMode(String cipherName) + { + this.cipherName = cipherName; + } + + public String getCipherName() + { + return cipherName; + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ColumnDecryptionContext.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ColumnDecryptionContext.java new file mode 100644 index 000000000000..c251c2149c7d --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ColumnDecryptionContext.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import org.apache.parquet.format.BlockCipher.Decryptor; + +import java.util.Arrays; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public record ColumnDecryptionContext(Decryptor dataDecryptor, Decryptor metadataDecryptor, byte[] fileAad) +{ + public ColumnDecryptionContext(Decryptor dataDecryptor, Decryptor metadataDecryptor, byte[] fileAad) + { + this.dataDecryptor = requireNonNull(dataDecryptor, "dataDecryptor is null"); + this.metadataDecryptor = requireNonNull(metadataDecryptor, "metadataDecryptor is null"); + this.fileAad = requireNonNull(fileAad, "fileAad is null"); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof ColumnDecryptionContext context)) { + return false; + } + return Objects.equals(dataDecryptor, context.dataDecryptor) + && Objects.equals(metadataDecryptor, context.metadataDecryptor) + && Objects.deepEquals(fileAad, context.fileAad); + } + + @Override + public int hashCode() + { + return Objects.hash(dataDecryptor, metadataDecryptor, Arrays.hashCode(fileAad)); + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/DecryptionKeyRetriever.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/DecryptionKeyRetriever.java new file mode 100644 index 000000000000..130a84534bbf --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/DecryptionKeyRetriever.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import org.apache.parquet.hadoop.metadata.ColumnPath; + +import java.util.Optional; + +/** + * Interface for classes retrieving encryption keys using the key metadata. + * Implementations must be thread-safe, if same {@link DecryptionKeyRetriever} object is passed to multiple file readers. + */ +public interface DecryptionKeyRetriever +{ + /** + * Returns key for a given column and the key metadata. Should return empty if user does not have access to the column key or key doesn't exist. + */ + Optional getColumnKey(ColumnPath columnPath, Optional keyMetadata); + + /** + * Returns key for a footer and the key metadata. Should return empty if user does not have access to the column key or key doesn't exist. + */ + Optional getFooterKey(Optional keyMetadata); +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/FileDecryptionContext.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/FileDecryptionContext.java new file mode 100644 index 000000000000..f77b8fa37f47 --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/FileDecryptionContext.java @@ -0,0 +1,183 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import io.airlift.log.Logger; +import io.trino.parquet.ParquetDataSourceId; +import org.apache.parquet.format.BlockCipher.Decryptor; +import org.apache.parquet.format.EncryptionAlgorithm; +import org.apache.parquet.hadoop.metadata.ColumnPath; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.primitives.Bytes.concat; +import static io.trino.parquet.ParquetValidationUtils.validateParquetCrypto; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class FileDecryptionContext +{ + private static final Logger log = Logger.get(FileDecryptionContext.class); + + private final Map> columnDecryptionContext = new HashMap<>(); + + private final ParquetDataSourceId dataSourceId; + private final DecryptionKeyRetriever keyRetriever; + private final EncryptionAlgorithm algorithm; + private final Optional footerKey; + private final byte[] fileAad; + + private AesGcmDecryptor aesGcmDecryptorWithFooterKey; + private AesCtrDecryptor aesCtrDecryptorWithFooterKey; + + public FileDecryptionContext(ParquetDataSourceId dataSourceId, FileDecryptionProperties fileDecryptionProperties, EncryptionAlgorithm algorithm, Optional footerKeyMetadata) + { + log.debug("File Decryptor. Algo: %s", algorithm); + + requireNonNull(fileDecryptionProperties, "fileDecryptionProperties is null"); + this.dataSourceId = requireNonNull(dataSourceId, "dataSourceId is null"); + this.keyRetriever = fileDecryptionProperties.getKeyRetriever(); + this.algorithm = requireNonNull(algorithm, "algorithm is null"); + + byte[] aadFileUnique; + boolean mustSupplyAadPrefix; + byte[] aadPrefixInFile = null; + + // Process encryption algorithm metadata + if (algorithm.isSetAES_GCM_V1()) { + if (algorithm.getAES_GCM_V1().isSetAad_prefix()) { + aadPrefixInFile = algorithm.getAES_GCM_V1().getAad_prefix(); + } + mustSupplyAadPrefix = algorithm.getAES_GCM_V1().isSupply_aad_prefix(); + aadFileUnique = algorithm.getAES_GCM_V1().getAad_file_unique(); + } + else if (algorithm.isSetAES_GCM_CTR_V1()) { + if (algorithm.getAES_GCM_CTR_V1().isSetAad_prefix()) { + aadPrefixInFile = algorithm.getAES_GCM_CTR_V1().getAad_prefix(); + } + mustSupplyAadPrefix = algorithm.getAES_GCM_CTR_V1().isSupply_aad_prefix(); + aadFileUnique = algorithm.getAES_GCM_CTR_V1().getAad_file_unique(); + } + else { + throw new UnsupportedOperationException(format("Unsupported algorithm: %s", algorithm)); + } + + // Determine AAD prefix, in-file AAD prefix takes precedence + Optional aadPrefix = Optional.ofNullable(aadPrefixInFile).or(fileDecryptionProperties::getAadPrefix); + validateParquetCrypto(!mustSupplyAadPrefix || aadPrefix.isPresent(), dataSourceId, "AAD prefix must be supplied"); + fileAad = aadPrefix.map(bytes -> concat(bytes, aadFileUnique)).orElse(aadFileUnique); + footerKey = fileDecryptionProperties.getKeyRetriever().getFooterKey(footerKeyMetadata); + } + + public Optional getColumnDecryptionContext(ColumnPath path) + { + Optional context = columnDecryptionContext.get(path); + checkArgument(context != null, "Column %s not found in decryption context", path); + return context; + } + + public Decryptor getFooterDecryptor() + { + validateParquetCrypto(footerKey.isPresent(), dataSourceId, "User does not have access to footer or footer key does not exists"); + return getThriftModuleDecryptor(Optional.empty()); + } + + public AesGcmEncryptor getFooterEncryptor() + { + validateParquetCrypto(footerKey.isPresent(), dataSourceId, "User does not have access to footer or footer key does not exists"); + return new AesGcmEncryptor(footerKey.get()); + } + + public byte[] getFileAad() + { + return this.fileAad; + } + + public void initPlaintextColumn(ColumnPath path) + { + log.debug("Column decryption (plaintext): %s", path); + setColumnDecryptionContext(path, Optional.empty()); + } + + public Optional initializeColumnCryptoMetadata(ColumnPath path, boolean encryptedWithFooterKey, Optional columnKeyMetadata) + { + log.debug("Column decryption (footer key): %s", path); + + Optional context; + if (encryptedWithFooterKey) { + if (footerKey.isEmpty()) { + // User does not have access to the footer key. Column is considered hidden. + setColumnDecryptionContext(path, Optional.empty()); + return Optional.empty(); + } + context = Optional.of(new ColumnDecryptionContext(getDataModuleDecryptor(Optional.empty()), getThriftModuleDecryptor(Optional.empty()), fileAad)); + } + else { + // Column is encrypted with column-specific key + Optional columnKeyBytes = requireNonNull(keyRetriever.getColumnKey(path, columnKeyMetadata), format("Column key for %s not found", path)); + if (columnKeyBytes.isEmpty()) { + // User does not have access to the column key. Column is considered hidden. + setColumnDecryptionContext(path, Optional.empty()); + return Optional.empty(); + } + context = Optional.of(new ColumnDecryptionContext(getDataModuleDecryptor(columnKeyBytes), getThriftModuleDecryptor(columnKeyBytes), fileAad)); + } + + setColumnDecryptionContext(path, context); + return context; + } + + private void setColumnDecryptionContext(ColumnPath path, Optional context) + { + checkArgument(!columnDecryptionContext.containsKey(path) || columnDecryptionContext.get(path).equals(context), "Mismatching column %s encryption context already exists in decryption context", path); + columnDecryptionContext.put(path, context); + } + + private Decryptor getDataModuleDecryptor(Optional columnKey) + { + if (algorithm.isSetAES_GCM_V1()) { + return getThriftModuleDecryptor(columnKey); + } + + // AES_GCM_CTR_V1 + if (columnKey.isEmpty()) { + // Decryptor with footer key + if (aesCtrDecryptorWithFooterKey == null) { + aesCtrDecryptorWithFooterKey = new AesCtrDecryptor(footerKey.get()); + } + return aesCtrDecryptorWithFooterKey; + } + else { + // Decryptor with column key + return new AesCtrDecryptor(columnKey.orElseThrow()); + } + } + + private Decryptor getThriftModuleDecryptor(Optional columnKey) + { + if (columnKey.isEmpty()) { + // Decryptor with footer key + if (aesGcmDecryptorWithFooterKey == null) { + aesGcmDecryptorWithFooterKey = new AesGcmDecryptor(footerKey.get()); + } + return aesGcmDecryptorWithFooterKey; + } + + // Decryptor with column key + return new AesGcmDecryptor(columnKey.get()); + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/FileDecryptionProperties.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/FileDecryptionProperties.java new file mode 100644 index 000000000000..574ca1252829 --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/FileDecryptionProperties.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class FileDecryptionProperties +{ + private final DecryptionKeyRetriever keyRetriever; + private final Optional aadPrefix; + private final boolean checkFooterIntegrity; + + private FileDecryptionProperties(DecryptionKeyRetriever keyRetriever, Optional aadPrefix, boolean checkFooterIntegrity) + { + this.keyRetriever = requireNonNull(keyRetriever, "keyRetriever is null"); + this.aadPrefix = requireNonNull(aadPrefix, "aadPrefix is null"); + this.checkFooterIntegrity = checkFooterIntegrity; + } + + public static Builder builder() + { + return new Builder(); + } + + public DecryptionKeyRetriever getKeyRetriever() + { + return keyRetriever; + } + + public Optional getAadPrefix() + { + return aadPrefix; + } + + public boolean isCheckFooterIntegrity() + { + return checkFooterIntegrity; + } + + public static class Builder + { + private DecryptionKeyRetriever keyRetriever; + private Optional aadPrefix = Optional.empty(); + private boolean checkFooterIntegrity = true; + + public Builder withKeyRetriever(DecryptionKeyRetriever keyRetriever) + { + this.keyRetriever = requireNonNull(keyRetriever, "keyRetriever is null"); + return this; + } + + public Builder withAadPrefix(byte[] aadPrefix) + { + this.aadPrefix = Optional.of(requireNonNull(aadPrefix, "aadPrefix is null")); + return this; + } + + public Builder withCheckFooterIntegrity(boolean checkFooterIntegrity) + { + this.checkFooterIntegrity = checkFooterIntegrity; + return this; + } + + public FileDecryptionProperties build() + { + return new FileDecryptionProperties(keyRetriever, aadPrefix, checkFooterIntegrity); + } + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ModuleType.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ModuleType.java new file mode 100644 index 000000000000..af56ad50a54c --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ModuleType.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +public enum ModuleType +{ + Footer((byte) 0), + ColumnMetaData((byte) 1), + DataPage((byte) 2), + DictionaryPage((byte) 3), + DataPageHeader((byte) 4), + DictionaryPageHeader((byte) 5), + ColumnIndex((byte) 6), + OffsetIndex((byte) 7), + BloomFilterHeader((byte) 8), + BloomFilterBitset((byte) 9); + + private final byte value; + + ModuleType(byte value) + { + this.value = value; + } + + public byte getValue() + { + return value; + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ParquetCryptoException.java b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ParquetCryptoException.java new file mode 100644 index 000000000000..32d18aea2b47 --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/crypto/ParquetCryptoException.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import com.google.errorprone.annotations.FormatMethod; +import io.trino.parquet.ParquetDataSourceId; + +import java.util.Optional; + +import static java.lang.String.format; + +public class ParquetCryptoException + extends RuntimeException +{ + @FormatMethod + public ParquetCryptoException(String messageFormat, Object... args) + { + super(formatMessage(Optional.empty(), messageFormat, args)); + } + + @FormatMethod + public ParquetCryptoException(Throwable cause, String messageFormat, Object... args) + { + super(formatMessage(Optional.empty(), messageFormat, args), cause); + } + + @FormatMethod + public ParquetCryptoException(ParquetDataSourceId dataSourceId, String messageFormat, Object... args) + { + super(formatMessage(Optional.of(dataSourceId), messageFormat, args)); + } + + private static String formatMessage(Optional dataSourceId, String messageFormat, Object[] args) + { + if (dataSourceId.isEmpty()) { + return "Parquet cryptographic error. " + format(messageFormat, args); + } + + return "Parquet cryptographic error. " + format(messageFormat, args) + " [" + dataSourceId + "]"; + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/BlockMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/BlockMetadata.java index 939bc399037e..c057a6207689 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/BlockMetadata.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/BlockMetadata.java @@ -17,8 +17,4 @@ public record BlockMetadata(long fileRowCountOffset, long rowCount, List columns) { - public long getStartingPos() - { - return columns().getFirst().getStartingPos(); - } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ColumnChunkMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ColumnChunkMetadata.java index 381260829869..62d3d9869d67 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ColumnChunkMetadata.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ColumnChunkMetadata.java @@ -23,9 +23,13 @@ import java.util.Set; +import static org.apache.parquet.column.Encoding.PLAIN_DICTIONARY; +import static org.apache.parquet.column.Encoding.RLE_DICTIONARY; + public abstract class ColumnChunkMetadata { - protected int rowGroupOrdinal = -1; + private int rowGroupOrdinal = -1; + private int columnOrdinal = -1; public static ColumnChunkMetadata get( ColumnPath path, @@ -76,6 +80,16 @@ public int getRowGroupOrdinal() return rowGroupOrdinal; } + public void setColumnOrdinal(int columnOrdinal) + { + this.columnOrdinal = columnOrdinal; + } + + public int getColumnOrdinal() + { + return columnOrdinal; + } + public long getStartingPos() { decryptIfNeeded(); @@ -194,6 +208,18 @@ public EncodingStats getEncodingStats() return encodingStats; } + public boolean hasDictionaryPage() + { + decryptIfNeeded(); + if (encodingStats != null) { + // ensure there is a dictionary page and that it is used to encode data pages + return encodingStats.hasDictionaryPages() && encodingStats.hasDictionaryEncodedPages(); + } + + Set encodings = properties.encodings(); + return (encodings.contains(PLAIN_DICTIONARY) || encodings.contains(RLE_DICTIONARY)); + } + @Override public String toString() { diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/HiddenColumnChunkMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/HiddenColumnChunkMetadata.java new file mode 100644 index 000000000000..95db2cb4ca9c --- /dev/null +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/HiddenColumnChunkMetadata.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.metadata; + +import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.crypto.ParquetCryptoException; +import org.apache.parquet.column.statistics.Statistics; +import org.apache.parquet.hadoop.metadata.ColumnPath; + +import static java.util.Objects.requireNonNull; + +public class HiddenColumnChunkMetadata + extends ColumnChunkMetadata +{ + private final ParquetDataSourceId dataSourceId; + private final ColumnPath path; + + public HiddenColumnChunkMetadata(ParquetDataSourceId dataSourceId, ColumnPath path) + { + super(null, null); + this.dataSourceId = requireNonNull(dataSourceId, "dataSourceId is null"); + this.path = requireNonNull(path, "path is null"); + } + + @Override + public ColumnPath getPath() + { + return path; + } + + @Override + public long getFirstDataPageOffset() + { + throw hiddenColumnException(); + } + + @Override + public long getDictionaryPageOffset() + { + throw hiddenColumnException(); + } + + @Override + public long getValueCount() + { + throw hiddenColumnException(); + } + + @Override + public long getTotalUncompressedSize() + { + throw hiddenColumnException(); + } + + @Override + public long getTotalSize() + { + throw hiddenColumnException(); + } + + @Override + public Statistics getStatistics() + { + throw hiddenColumnException(); + } + + public static boolean isHiddenColumn(ColumnChunkMetadata column) + { + return column instanceof HiddenColumnChunkMetadata; + } + + private ParquetCryptoException hiddenColumnException() + { + return new ParquetCryptoException(dataSourceId, "User does not have access to column: %s or column key does not exists", path); + } + + @Override + public String toString() + { + return "HiddenColumnChunkMetadata{dataSourceId=" + dataSourceId + ", path=" + path + "}"; + } +} diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java index f0c640bd0dd8..6d39b434d6d9 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/metadata/ParquetMetadata.java @@ -19,14 +19,21 @@ import io.airlift.log.Logger; import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.crypto.AesCipherUtils; +import io.trino.parquet.crypto.ColumnDecryptionContext; +import io.trino.parquet.crypto.FileDecryptionContext; +import io.trino.parquet.crypto.ModuleType; import io.trino.parquet.reader.MetadataReader; import org.apache.parquet.column.Encoding; import org.apache.parquet.format.ColumnChunk; +import org.apache.parquet.format.ColumnCryptoMetaData; import org.apache.parquet.format.ColumnMetaData; +import org.apache.parquet.format.EncryptionWithColumnKey; import org.apache.parquet.format.FileMetaData; import org.apache.parquet.format.KeyValue; import org.apache.parquet.format.RowGroup; import org.apache.parquet.format.SchemaElement; +import org.apache.parquet.format.Util; import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.parquet.schema.LogicalTypeAnnotation; @@ -35,6 +42,8 @@ import org.apache.parquet.schema.Type; import org.apache.parquet.schema.Types; +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -54,6 +63,7 @@ import static io.trino.parquet.ParquetMetadataConverter.toColumnIndexReference; import static io.trino.parquet.ParquetMetadataConverter.toOffsetIndexReference; import static io.trino.parquet.ParquetValidationUtils.validateParquet; +import static io.trino.parquet.ParquetValidationUtils.validateParquetCrypto; import static java.util.Objects.requireNonNull; public class ParquetMetadata @@ -63,8 +73,9 @@ public class ParquetMetadata private final FileMetaData parquetMetadata; private final ParquetDataSourceId dataSourceId; private final FileMetadata fileMetadata; + private final Optional decryptionContext; - public ParquetMetadata(FileMetaData parquetMetadata, ParquetDataSourceId dataSourceId) + public ParquetMetadata(FileMetaData parquetMetadata, ParquetDataSourceId dataSourceId, Optional decryptionContext) throws ParquetCorruptionException { this.fileMetadata = new FileMetadata( @@ -73,6 +84,7 @@ public ParquetMetadata(FileMetaData parquetMetadata, ParquetDataSourceId dataSou parquetMetadata.getCreated_by()); this.parquetMetadata = parquetMetadata; this.dataSourceId = requireNonNull(dataSourceId, "dataSourceId is null"); + this.decryptionContext = requireNonNull(decryptionContext, "decryptionContext is null"); } public FileMetadata getFileMetaData() @@ -80,6 +92,11 @@ public FileMetadata getFileMetaData() return fileMetadata; } + public Optional getDecryptionContext() + { + return decryptionContext; + } + @Override public String toString() { @@ -89,13 +106,13 @@ public String toString() } public List getBlocks() - throws ParquetCorruptionException + throws IOException { return getBlocks(0, Long.MAX_VALUE); } public List getBlocks(long splitStart, long splitLength) - throws ParquetCorruptionException + throws IOException { List schema = parquetMetadata.getSchema(); validateParquet(!schema.isEmpty(), dataSourceId, "Schema is empty"); @@ -114,21 +131,81 @@ public List getBlocks(long splitStart, long splitLength) List columns = rowGroup.getColumns(); validateParquet(!columns.isEmpty(), dataSourceId, "No columns in row group: %s", rowGroup); String filePath = columns.get(0).getFile_path(); - long rowGroupStart = getRowGroupStart(columns, messageType); - boolean splitContainsRowGroup = splitStart <= rowGroupStart && rowGroupStart < splitStart + splitLength; - if (!splitContainsRowGroup) { - continue; - } ImmutableList.Builder columnMetadataBuilder = ImmutableList.builderWithExpectedSize(columns.size()); + int columnOrdinal = -1; + boolean splitContainsRowGroup = true; for (ColumnChunk columnChunk : columns) { + columnOrdinal++; validateParquet( (filePath == null && columnChunk.getFile_path() == null) || (filePath != null && filePath.equals(columnChunk.getFile_path())), dataSourceId, "all column chunks of the same row group must be in the same file"); - ColumnChunkMetadata column = toColumnChunkMetadata(columnChunk, parquetMetadata.getCreated_by(), messageType); + ColumnCryptoMetaData cryptoMetaData = columnChunk.getCrypto_metadata(); + ColumnMetaData metaData; + ColumnPath columnPath; + if (cryptoMetaData == null) { + // Plaintext column + metaData = columnChunk.getMeta_data(); + columnPath = getPath(metaData.getPath_in_schema()); + decryptionContext.ifPresent(context -> context.initPlaintextColumn(columnPath)); + } + else { + validateParquetCrypto(decryptionContext.isPresent(), dataSourceId, "Column is encrypted, but no decryption context"); + if (cryptoMetaData.isSetENCRYPTION_WITH_FOOTER_KEY()) { + // Column encrypted with footer key + validateParquetCrypto(columnChunk.getMeta_data() != null, dataSourceId, "Column metadata is null"); + metaData = columnChunk.getMeta_data(); + columnPath = getPath(metaData.getPath_in_schema()); + decryptionContext.get().initializeColumnCryptoMetadata(columnPath, true, Optional.empty()); + } + else { + // Column encrypted with column key + EncryptionWithColumnKey columnKeyStruct = cryptoMetaData.getENCRYPTION_WITH_COLUMN_KEY(); + columnPath = getPath(columnKeyStruct.getPath_in_schema()); + Optional decryptedMetadata = decryptColumnMetadata(columnPath, rowGroup, columnKeyStruct.getKey_metadata(), columnChunk, decryptionContext.get(), columnOrdinal); + if (decryptedMetadata.isEmpty()) { + // User does not have access to the column key. Column is considered hidden. + columnMetadataBuilder.add(new HiddenColumnChunkMetadata(dataSourceId, columnPath)); + validateParquetCrypto(columnOrdinal != 0, dataSourceId, "First column of a row group is encrypted with an unknown column key. Cannot determine row group starting position."); + continue; + } + metaData = decryptedMetadata.get(); + } + } + + PrimitiveType primitiveType = messageType.getType(columnPath.toArray()).asPrimitiveType(); + ColumnChunkMetadata column = ColumnChunkMetadata.get( + columnPath, + primitiveType, + CompressionCodecName.fromParquet(metaData.codec), + convertEncodingStats(metaData.encoding_stats), + readEncodings(metaData.encodings), + MetadataReader.readStats(Optional.ofNullable(parquetMetadata.getCreated_by()), Optional.ofNullable(metaData.statistics), primitiveType), + metaData.data_page_offset, + metaData.dictionary_page_offset, + metaData.num_values, + metaData.total_compressed_size, + metaData.total_uncompressed_size); + column.setColumnIndexReference(toColumnIndexReference(columnChunk)); + column.setOffsetIndexReference(toOffsetIndexReference(columnChunk)); + column.setBloomFilterOffset(metaData.bloom_filter_offset); + if (rowGroup.isSetOrdinal()) { + column.setRowGroupOrdinal(rowGroup.getOrdinal()); + } + column.setColumnOrdinal(columnOrdinal); columnMetadataBuilder.add(column); + + // Skip row group if it doesn't overlap the split. Only first column starting position matches row group start and can be used for the check. + long rowGroupStart = getRowGroupStart(column); + splitContainsRowGroup = columnOrdinal != 0 || (splitStart <= rowGroupStart && rowGroupStart < splitStart + splitLength); + if (!splitContainsRowGroup) { + break; + } + } + if (!splitContainsRowGroup) { + continue; } blocks.add(new BlockMetadata(fileRowCountOffset, rowGroup.getNum_rows(), columnMetadataBuilder.build())); } @@ -143,38 +220,11 @@ public FileMetaData getParquetMetadata() return parquetMetadata; } - private static long getRowGroupStart(List columns, MessageType messageType) + private static long getRowGroupStart(ColumnChunkMetadata column) { // Note: Do not rely on org.apache.parquet.format.RowGroup.getFile_offset or org.apache.parquet.format.ColumnChunk.getFile_offset // because some versions of parquet-cpp-arrow (and potentially other writers) set it incorrectly - ColumnChunkMetadata columnChunkMetadata = toColumnChunkMetadata(columns.getFirst(), null, messageType); - return columnChunkMetadata.getStartingPos(); - } - - private static ColumnChunkMetadata toColumnChunkMetadata(ColumnChunk columnChunk, String createdBy, MessageType messageType) - { - ColumnMetaData metaData = columnChunk.meta_data; - String[] path = metaData.path_in_schema.stream() - .map(value -> value.toLowerCase(Locale.ENGLISH)) - .toArray(String[]::new); - ColumnPath columnPath = ColumnPath.get(path); - PrimitiveType primitiveType = messageType.getType(columnPath.toArray()).asPrimitiveType(); - ColumnChunkMetadata column = ColumnChunkMetadata.get( - columnPath, - primitiveType, - CompressionCodecName.fromParquet(metaData.codec), - convertEncodingStats(metaData.encoding_stats), - readEncodings(metaData.encodings), - MetadataReader.readStats(Optional.ofNullable(createdBy), Optional.ofNullable(metaData.statistics), primitiveType), - metaData.data_page_offset, - metaData.dictionary_page_offset, - metaData.num_values, - metaData.total_compressed_size, - metaData.total_uncompressed_size); - column.setColumnIndexReference(toColumnIndexReference(columnChunk)); - column.setOffsetIndexReference(toOffsetIndexReference(columnChunk)); - column.setBloomFilterOffset(metaData.bloom_filter_offset); - return column; + return column.getStartingPos(); } private static MessageType readParquetSchema(List schema) @@ -248,6 +298,31 @@ private static void readTypeSchema(Types.GroupBuilder builder, Iterator decryptColumnMetadata(ColumnPath columnPath, RowGroup rowGroup, byte[] columnKeyMetadata, ColumnChunk columnChunk, FileDecryptionContext decryptionContext, int columnOrdinal) + throws IOException + { + byte[] encryptedMetadataBuffer = columnChunk.getEncrypted_column_metadata(); + + // Decrypt the ColumnMetaData + Optional columnDecryptionContext = decryptionContext.initializeColumnCryptoMetadata(columnPath, false, Optional.ofNullable(columnKeyMetadata)); + if (columnDecryptionContext.isEmpty()) { + return Optional.empty(); + } + + ByteArrayInputStream tempInputStream = new ByteArrayInputStream(encryptedMetadataBuffer); + byte[] columnMetaDataAAD = AesCipherUtils.createModuleAAD(decryptionContext.getFileAad(), ModuleType.ColumnMetaData, rowGroup.ordinal, columnOrdinal, -1); + return Optional.of(Util.readColumnMetaData(tempInputStream, columnDecryptionContext.get().metadataDecryptor(), columnMetaDataAAD)); + } + + private static ColumnPath getPath(List pathInSchema) + { + requireNonNull(pathInSchema, "pathInSchema is null"); + String[] path = pathInSchema.stream() + .map(value -> value.toLowerCase(Locale.ENGLISH)) + .toArray(String[]::new); + return ColumnPath.get(path); + } + private static Set readEncodings(List encodings) { Set columnEncodings = new HashSet<>(); diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java b/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java index 3230c7190a0a..1a34a62659b3 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/predicate/PredicateUtils.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableSet; import io.airlift.slice.Slice; import io.airlift.slice.SliceInput; +import io.airlift.slice.Slices; import io.trino.parquet.BloomFilterStore; import io.trino.parquet.DictionaryPage; import io.trino.parquet.ParquetCorruptionException; @@ -25,6 +26,10 @@ import io.trino.parquet.ParquetDataSourceId; import io.trino.parquet.ParquetEncoding; import io.trino.parquet.ParquetReaderOptions; +import io.trino.parquet.crypto.AesCipherUtils; +import io.trino.parquet.crypto.ColumnDecryptionContext; +import io.trino.parquet.crypto.FileDecryptionContext; +import io.trino.parquet.crypto.ModuleType; import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.parquet.metadata.ParquetMetadata; @@ -36,10 +41,10 @@ import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.statistics.Statistics; +import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.DictionaryPageHeader; import org.apache.parquet.format.PageHeader; import org.apache.parquet.format.PageType; -import org.apache.parquet.format.Util; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import org.apache.parquet.internal.filter2.columnindex.ColumnIndexStore; import org.apache.parquet.io.ParquetDecodingException; @@ -69,6 +74,7 @@ import static java.lang.Math.toIntExact; import static java.lang.String.format; import static java.util.Objects.requireNonNull; +import static org.apache.parquet.format.Util.readPageHeader; public final class PredicateUtils { @@ -142,7 +148,8 @@ public static boolean predicateMatches( Optional columnIndexStore, Optional bloomFilterStore, DateTimeZone timeZone, - int domainCompactionThreshold) + int domainCompactionThreshold, + Optional decryptionContext) throws IOException { if (columnsMetadata.getRowCount() == 0) { @@ -177,7 +184,8 @@ public static boolean predicateMatches( dataSource, descriptorsByPath, ImmutableSet.copyOf(candidateColumns.get()), - columnIndexStore); + columnIndexStore, + decryptionContext); } public static List getFilteredRowGroups( @@ -198,8 +206,8 @@ public static List getFilteredRowGroups( for (int i = 0; i < parquetTupleDomains.size(); i++) { TupleDomain parquetTupleDomain = parquetTupleDomains.get(i); TupleDomainParquetPredicate parquetPredicate = parquetPredicates.get(i); - Optional columnIndex = getColumnIndexStore(dataSource, block, descriptorsByPath, parquetTupleDomain, options); - Optional bloomFilterStore = getBloomFilterStore(dataSource, block, parquetTupleDomain, options); + Optional columnIndex = getColumnIndexStore(dataSource, block, descriptorsByPath, parquetTupleDomain, options, parquetMetadata.getDecryptionContext()); + Optional bloomFilterStore = getBloomFilterStore(dataSource, block, parquetTupleDomain, options, parquetMetadata.getDecryptionContext()); PrunedBlockMetadata columnsMetadata = createPrunedColumnsMetadata(block, dataSource.getId(), descriptorsByPath); if (predicateMatches( parquetPredicate, @@ -210,7 +218,8 @@ public static List getFilteredRowGroups( columnIndex, bloomFilterStore, timeZone, - domainCompactionThreshold)) { + domainCompactionThreshold, + parquetMetadata.getDecryptionContext())) { rowGroupInfoBuilder.add(new RowGroupInfo(columnsMetadata, block.fileRowCountOffset(), columnIndex)); break; } @@ -250,7 +259,8 @@ private static boolean dictionaryPredicatesMatch( ParquetDataSource dataSource, Map, ColumnDescriptor> descriptorsByPath, Set candidateColumns, - Optional columnIndexStore) + Optional columnIndexStore, + Optional decryptionContext) throws IOException { for (ColumnDescriptor descriptor : descriptorsByPath.values()) { @@ -265,7 +275,7 @@ private static boolean dictionaryPredicatesMatch( if (!parquetPredicate.matches(new DictionaryDescriptor( descriptor, nullAllowed, - readDictionaryPage(dataSource, columnMetaData, columnIndexStore)))) { + readDictionaryPage(dataSource, columnMetaData, columnIndexStore, decryptionContext)))) { return false; } } @@ -276,7 +286,8 @@ private static boolean dictionaryPredicatesMatch( private static Optional readDictionaryPage( ParquetDataSource dataSource, ColumnChunkMetadata columnMetaData, - Optional columnIndexStore) + Optional columnIndexStore, + Optional decryptionContext) throws IOException { int dictionaryPageSize; @@ -300,7 +311,8 @@ private static Optional readDictionaryPage( } // Get the dictionary page header and the dictionary in single read Slice buffer = dataSource.readFully(columnMetaData.getStartingPos(), dictionaryPageSize); - return readPageHeaderWithData(buffer.getInput()).map(data -> decodeDictionaryPage(dataSource.getId(), data, columnMetaData)); + return readPageHeaderWithData(buffer.getInput(), columnMetaData, decryptionContext) + .map(data -> decodeDictionaryPage(dataSource.getId(), data, columnMetaData, decryptionContext)); } private static Optional getDictionaryPageSize(ColumnIndexStore columnIndexStore, ColumnChunkMetadata columnMetaData) @@ -317,11 +329,20 @@ private static Optional getDictionaryPageSize(ColumnIndexStore columnIn return Optional.empty(); } - private static Optional readPageHeaderWithData(SliceInput inputStream) + private static Optional readPageHeaderWithData(SliceInput inputStream, ColumnChunkMetadata columnMetaData, Optional decryptionContext) { - PageHeader pageHeader; + Optional columnContext = decryptionContext.flatMap(context -> context.getColumnDecryptionContext(columnMetaData.getPath())); + BlockCipher.Decryptor decryptor = null; + byte[] headerAad = null; + if (columnContext.isPresent()) { + decryptor = columnContext.map(ColumnDecryptionContext::metadataDecryptor).orElse(null); + byte[] fileAad = decryptionContext.get().getFileAad(); + headerAad = AesCipherUtils.createModuleAAD(fileAad, ModuleType.DictionaryPageHeader, columnMetaData.getRowGroupOrdinal(), columnMetaData.getColumnOrdinal(), -1); + } + + final PageHeader pageHeader; try { - pageHeader = Util.readPageHeader(inputStream); + pageHeader = readPageHeader(inputStream, decryptor, headerAad); } catch (IOException e) { throw new UncheckedIOException(e); @@ -339,7 +360,7 @@ private static Optional readPageHeaderWithData(SliceInput in inputStream.readSlice(pageHeader.getCompressed_page_size()))); } - private static DictionaryPage decodeDictionaryPage(ParquetDataSourceId dataSourceId, PageHeaderWithData pageHeaderWithData, ColumnChunkMetadata chunkMetaData) + private static DictionaryPage decodeDictionaryPage(ParquetDataSourceId dataSourceId, PageHeaderWithData pageHeaderWithData, ColumnChunkMetadata chunkMetaData, Optional decryptionContext) { PageHeader pageHeader = pageHeaderWithData.pageHeader(); DictionaryPageHeader dicHeader = pageHeader.getDictionary_page_header(); @@ -347,8 +368,20 @@ private static DictionaryPage decodeDictionaryPage(ParquetDataSourceId dataSourc int dictionarySize = dicHeader.getNum_values(); Slice compressedData = pageHeaderWithData.compressedData(); + Slice maybeDecrypted = compressedData; + Optional columnContext = decryptionContext.flatMap(context -> context.getColumnDecryptionContext(chunkMetaData.getPath())); + if (columnContext.isPresent()) { + byte[] aad = AesCipherUtils.createModuleAAD( + columnContext.get().fileAad(), + ModuleType.DictionaryPage, + chunkMetaData.getRowGroupOrdinal(), + chunkMetaData.getColumnOrdinal(), + -1); + byte[] plain = columnContext.get().dataDecryptor().decrypt(compressedData.getBytes(), aad); + maybeDecrypted = Slices.wrappedBuffer(plain); + } try { - return new DictionaryPage(decompress(dataSourceId, chunkMetaData.getCodec().getParquetCompressionCodec(), compressedData, pageHeader.getUncompressed_page_size()), dictionarySize, encoding); + return new DictionaryPage(decompress(dataSourceId, chunkMetaData.getCodec().getParquetCompressionCodec(), maybeDecrypted, pageHeader.getUncompressed_page_size()), dictionarySize, encoding); } catch (IOException e) { throw new ParquetDecodingException("Could not decode the dictionary for " + chunkMetaData.getPath(), e); diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java index 369ce467e131..8808d592995a 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/MetadataReader.java @@ -14,56 +14,68 @@ package io.trino.parquet.reader; import io.airlift.slice.Slice; +import io.airlift.slice.SliceInput; import io.airlift.slice.Slices; import io.airlift.units.DataSize; import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.ParquetDataSource; import io.trino.parquet.ParquetDataSourceId; import io.trino.parquet.ParquetWriteValidation; +import io.trino.parquet.crypto.AesCipherUtils; +import io.trino.parquet.crypto.AesGcmEncryptor; +import io.trino.parquet.crypto.FileDecryptionContext; +import io.trino.parquet.crypto.FileDecryptionProperties; import io.trino.parquet.metadata.FileMetadata; import io.trino.parquet.metadata.ParquetMetadata; import org.apache.parquet.CorruptStatistics; import org.apache.parquet.column.statistics.BinaryStatistics; +import org.apache.parquet.format.BlockCipher.Decryptor; +import org.apache.parquet.format.FileCryptoMetaData; import org.apache.parquet.format.FileMetaData; import org.apache.parquet.format.Statistics; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.PrimitiveType; import java.io.IOException; -import java.io.InputStream; import java.util.Arrays; import java.util.Optional; import static io.trino.parquet.ParquetMetadataConverter.fromParquetStatistics; import static io.trino.parquet.ParquetValidationUtils.validateParquet; +import static io.trino.parquet.ParquetValidationUtils.validateParquetCrypto; +import static io.trino.parquet.crypto.AesCipherUtils.GCM_TAG_LENGTH; +import static io.trino.parquet.crypto.AesCipherUtils.NONCE_LENGTH; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; import static java.lang.Math.min; import static java.lang.Math.toIntExact; +import static java.lang.System.arraycopy; +import static org.apache.parquet.format.Util.readFileCryptoMetaData; import static org.apache.parquet.format.Util.readFileMetaData; public final class MetadataReader { private static final Slice MAGIC = Slices.utf8Slice("PAR1"); + private static final Slice EMAGIC = Slices.utf8Slice("PARE"); private static final int POST_SCRIPT_SIZE = Integer.BYTES + MAGIC.length(); // Typical 1GB files produced by Trino were found to have footer size between 30-40KB private static final int EXPECTED_FOOTER_SIZE = 48 * 1024; private MetadataReader() {} - public static ParquetMetadata readFooter(ParquetDataSource dataSource) + public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional fileDecryptionProperties) throws IOException { - return readFooter(dataSource, Optional.empty(), Optional.empty()); + return readFooter(dataSource, Optional.empty(), Optional.empty(), fileDecryptionProperties); } - public static ParquetMetadata readFooter(ParquetDataSource dataSource, DataSize maxFooterReadSize) + public static ParquetMetadata readFooter(ParquetDataSource dataSource, DataSize maxFooterReadSize, Optional fileDecryptionProperties) throws IOException { - return readFooter(dataSource, Optional.of(maxFooterReadSize), Optional.empty()); + return readFooter(dataSource, Optional.of(maxFooterReadSize), Optional.empty(), fileDecryptionProperties); } - public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional maxFooterReadSize, Optional parquetWriteValidation) + public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional maxFooterReadSize, Optional parquetWriteValidation, Optional fileDecryptionProperties) throws IOException { // Parquet File Layout: @@ -82,8 +94,9 @@ public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional< Slice buffer = dataSource.readTail(toIntExact(expectedReadSize)); Slice magic = buffer.slice(buffer.length() - MAGIC.length(), MAGIC.length()); - validateParquet(MAGIC.equals(magic), dataSource.getId(), "Expected magic number: %s got: %s", MAGIC.toStringUtf8(), magic.toStringUtf8()); + validateParquet(MAGIC.equals(magic) || EMAGIC.equals(magic), dataSource.getId(), "Expected magic number: %s or %s got: %s", MAGIC.toStringUtf8(), EMAGIC.toStringUtf8(), magic.toStringUtf8()); + boolean encryptedFooterMode = EMAGIC.equals(magic); int metadataLength = buffer.getInt(buffer.length() - POST_SCRIPT_SIZE); long metadataIndex = estimatedFileSize - POST_SCRIPT_SIZE - metadataLength; validateParquet( @@ -104,10 +117,32 @@ public static ParquetMetadata readFooter(ParquetDataSource dataSource, Optional< // initial read was not large enough, so just read again with the correct size buffer = dataSource.readTail(completeFooterSize); } - InputStream metadataStream = buffer.slice(buffer.length() - completeFooterSize, metadataLength).getInput(); + SliceInput metadataStream = buffer.slice(buffer.length() - completeFooterSize, metadataLength).getInput(); - FileMetaData fileMetaData = readFileMetaData(metadataStream); - ParquetMetadata parquetMetadata = new ParquetMetadata(fileMetaData, dataSource.getId()); + Optional decryptionContext = Optional.empty(); + Decryptor footerDecryptor = null; + byte[] aad = null; + + if (encryptedFooterMode) { + validateParquetCrypto(fileDecryptionProperties.isPresent(), dataSource.getId(), "fileDecryptionProperties cannot be null when encryptedFooterMode is true"); + FileCryptoMetaData fileCryptoMetaData = readFileCryptoMetaData(metadataStream); + validateParquetCrypto(fileCryptoMetaData != null, dataSource.getId(), "FileCryptoMetaData cannot be null when encryptedFooterMode is true"); + decryptionContext = Optional.of(new FileDecryptionContext(dataSource.getId(), fileDecryptionProperties.get(), fileCryptoMetaData.getEncryption_algorithm(), Optional.ofNullable(fileCryptoMetaData.getKey_metadata()))); + footerDecryptor = decryptionContext.get().getFooterDecryptor(); + aad = AesCipherUtils.createFooterAAD(decryptionContext.get().getFileAad()); + } + + FileMetaData fileMetaData = readFileMetaData(metadataStream, footerDecryptor, aad); + if (!encryptedFooterMode && fileDecryptionProperties.isPresent() && fileMetaData.isSetEncryption_algorithm()) { + // footer is not encrypted, but some columns might be encrypted + decryptionContext = Optional.of(new FileDecryptionContext(dataSource.getId(), fileDecryptionProperties.get(), fileMetaData.getEncryption_algorithm(), Optional.ofNullable(fileMetaData.getFooter_signing_key_metadata()))); + if (fileDecryptionProperties.get().isCheckFooterIntegrity()) { + // verify footer integrity + verifyFooterIntegrity(dataSource, metadataStream, decryptionContext.get(), metadataLength); + } + } + + ParquetMetadata parquetMetadata = new ParquetMetadata(fileMetaData, dataSource.getId(), decryptionContext); validateFileMetadata(dataSource.getId(), parquetMetadata.getFileMetaData(), parquetWriteValidation); return parquetMetadata; } @@ -219,4 +254,26 @@ private static void validateFileMetadata(ParquetDataSourceId dataSourceId, FileM Optional.ofNullable(fileMetaData.getKeyValueMetaData().get("writer.time.zone"))); writeValidation.validateColumns(dataSourceId, fileMetaData.getSchema()); } + + private static void verifyFooterIntegrity(ParquetDataSource dataSource, SliceInput metadataStream, FileDecryptionContext decryptionContext, int metadataLength) + { + byte[] nonce = new byte[NONCE_LENGTH]; + metadataStream.read(nonce); + + byte[] gcmTag = new byte[GCM_TAG_LENGTH]; + metadataStream.read(gcmTag); + + // read only the serialized footer without the tags + int footerSignatureLength = NONCE_LENGTH + GCM_TAG_LENGTH; + byte[] footer = new byte[metadataLength - footerSignatureLength]; + metadataStream.setPosition(0); + metadataStream.read(footer, 0, footer.length); + byte[] signedFooterAAD = AesCipherUtils.createFooterAAD(decryptionContext.getFileAad()); + + AesGcmEncryptor footerSigner = decryptionContext.getFooterEncryptor(); + byte[] encryptedFooterBytes = footerSigner.encrypt(false, footer, nonce, signedFooterAAD); + byte[] calculatedTag = new byte[GCM_TAG_LENGTH]; + arraycopy(encryptedFooterBytes, encryptedFooterBytes.length - GCM_TAG_LENGTH, calculatedTag, 0, GCM_TAG_LENGTH); + validateParquetCrypto(Arrays.equals(gcmTag, calculatedTag), dataSource.getId(), "Signature mismatch in plaintext footer"); + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java index d8ec35c52fbe..0d9c85d8727e 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/PageReader.java @@ -16,17 +16,24 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterators; import com.google.common.collect.PeekingIterator; +import io.airlift.slice.Slice; import io.trino.parquet.DataPage; import io.trino.parquet.DataPageV1; import io.trino.parquet.DataPageV2; import io.trino.parquet.DictionaryPage; import io.trino.parquet.Page; import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.crypto.AesCipherUtils; +import io.trino.parquet.crypto.ColumnDecryptionContext; +import io.trino.parquet.crypto.FileDecryptionContext; +import io.trino.parquet.crypto.ModuleType; import io.trino.parquet.metadata.ColumnChunkMetadata; import jakarta.annotation.Nullable; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.statistics.Statistics; +import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.CompressionCodec; +import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.internal.column.columnindex.OffsetIndex; import java.io.IOException; @@ -35,6 +42,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; +import static io.airlift.slice.Slices.wrappedBuffer; import static io.trino.parquet.ParquetCompressionUtils.decompress; import static io.trino.parquet.ParquetReaderUtils.isOnlyDictionaryEncodingPages; import static java.util.Objects.requireNonNull; @@ -46,9 +54,14 @@ public final class PageReader private final boolean hasOnlyDictionaryEncodedPages; private final boolean hasNoNulls; private final PeekingIterator compressedPages; + private final Optional blockDecryptor; private boolean dictionaryAlreadyRead; private int dataPageReadCount; + @Nullable + private byte[] dataPageAad; + @Nullable + private byte[] dictionaryPageAad; public static PageReader createPageReader( ParquetDataSourceId dataSourceId, @@ -56,7 +69,8 @@ public static PageReader createPageReader( ColumnChunkMetadata metadata, ColumnDescriptor columnDescriptor, @Nullable OffsetIndex offsetIndex, - Optional fileCreatedBy) + Optional fileCreatedBy, + Optional decryptionContext) { // Parquet schema may specify a column definition as OPTIONAL even though there are no nulls in the actual data. // Row-group column statistics can be used to identify such cases and switch to faster non-nullable read @@ -64,20 +78,25 @@ public static PageReader createPageReader( Statistics columnStatistics = metadata.getStatistics(); boolean hasNoNulls = columnStatistics != null && columnStatistics.getNumNulls() == 0; boolean hasOnlyDictionaryEncodedPages = isOnlyDictionaryEncodingPages(metadata); + Optional columnDecryptionContext = decryptionContext.flatMap(context -> context.getColumnDecryptionContext(ColumnPath.get(columnDescriptor.getPath()))); ParquetColumnChunkIterator compressedPages = new ParquetColumnChunkIterator( dataSourceId, fileCreatedBy, columnDescriptor, metadata, columnChunk, - offsetIndex); + offsetIndex, + columnDecryptionContext); return new PageReader( dataSourceId, metadata.getCodec().getParquetCompressionCodec(), compressedPages, hasOnlyDictionaryEncodedPages, - hasNoNulls); + hasNoNulls, + columnDecryptionContext, + metadata.getRowGroupOrdinal(), + metadata.getColumnOrdinal()); } @VisibleForTesting @@ -86,13 +105,21 @@ public PageReader( CompressionCodec codec, Iterator compressedPages, boolean hasOnlyDictionaryEncodedPages, - boolean hasNoNulls) + boolean hasNoNulls, + Optional decryptionContext, + int rowGroupOrdinal, + int columnOrdinal) { this.dataSourceId = requireNonNull(dataSourceId, "dataSourceId is null"); this.codec = codec; this.compressedPages = Iterators.peekingIterator(compressedPages); this.hasOnlyDictionaryEncodedPages = hasOnlyDictionaryEncodedPages; this.hasNoNulls = hasNoNulls; + this.blockDecryptor = decryptionContext.map(ColumnDecryptionContext::dataDecryptor); + if (blockDecryptor.isPresent()) { + dataPageAad = AesCipherUtils.createModuleAAD(decryptionContext.get().fileAad(), ModuleType.DataPage, rowGroupOrdinal, columnOrdinal, 0); + dictionaryPageAad = AesCipherUtils.createModuleAAD(decryptionContext.get().fileAad(), ModuleType.DictionaryPage, rowGroupOrdinal, columnOrdinal, -1); + } } public boolean hasNoNulls() @@ -114,18 +141,20 @@ public DataPage readPage() checkState(compressedPage instanceof DataPage, "Found page %s instead of a DataPage", compressedPage); dataPageReadCount++; try { + if (blockDecryptor.isPresent()) { + AesCipherUtils.quickUpdatePageAAD(dataPageAad, ((DataPage) compressedPage).getPageIndex()); + } + Slice slice = decryptSliceIfNeeded(compressedPage.getSlice(), dataPageAad); if (compressedPage instanceof DataPageV1 dataPageV1) { - if (!arePagesCompressed()) { - return dataPageV1; - } return new DataPageV1( - decompress(dataSourceId, codec, dataPageV1.getSlice(), dataPageV1.getUncompressedSize()), + !arePagesCompressed() ? slice : decompress(dataSourceId, codec, slice, dataPageV1.getUncompressedSize()), dataPageV1.getValueCount(), dataPageV1.getUncompressedSize(), dataPageV1.getFirstRowIndex(), dataPageV1.getRepetitionLevelEncoding(), dataPageV1.getDefinitionLevelEncoding(), - dataPageV1.getValueEncoding()); + dataPageV1.getValueEncoding(), + dataPageV1.getPageIndex()); } DataPageV2 dataPageV2 = (DataPageV2) compressedPage; if (!dataPageV2.isCompressed()) { @@ -141,11 +170,12 @@ public DataPage readPage() dataPageV2.getRepetitionLevels(), dataPageV2.getDefinitionLevels(), dataPageV2.getDataEncoding(), - decompress(dataSourceId, codec, dataPageV2.getSlice(), uncompressedSize), + decompress(dataSourceId, codec, slice, uncompressedSize), dataPageV2.getUncompressedSize(), dataPageV2.getFirstRowIndex(), dataPageV2.getStatistics(), - false); + false, + dataPageV2.getPageIndex()); } catch (IOException e) { throw new RuntimeException("Could not decompress page", e); @@ -162,8 +192,9 @@ public DictionaryPage readDictionaryPage() } try { DictionaryPage compressedDictionaryPage = (DictionaryPage) compressedPages.next(); + Slice slice = decryptSliceIfNeeded(compressedDictionaryPage.getSlice(), dictionaryPageAad); return new DictionaryPage( - decompress(dataSourceId, codec, compressedDictionaryPage.getSlice(), compressedDictionaryPage.getUncompressedSize()), + decompress(dataSourceId, codec, slice, compressedDictionaryPage.getUncompressedSize()), compressedDictionaryPage.getDictionarySize(), compressedDictionaryPage.getEncoding()); } @@ -199,4 +230,14 @@ private void verifyDictionaryPageRead() { checkArgument(dictionaryAlreadyRead, "Dictionary has to be read first"); } + + private Slice decryptSliceIfNeeded(Slice slice, byte[] aad) + throws IOException + { + if (blockDecryptor.isEmpty()) { + return slice; + } + byte[] plainText = blockDecryptor.get().decrypt(slice.getBytes(), aad); + return wrappedBuffer(plainText); + } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetColumnChunkIterator.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetColumnChunkIterator.java index 235c1b2d3d76..6a67adcd44a6 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetColumnChunkIterator.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetColumnChunkIterator.java @@ -19,10 +19,14 @@ import io.trino.parquet.Page; import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.crypto.AesCipherUtils; +import io.trino.parquet.crypto.ColumnDecryptionContext; +import io.trino.parquet.crypto.ModuleType; import io.trino.parquet.metadata.ColumnChunkMetadata; import jakarta.annotation.Nullable; import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; +import org.apache.parquet.format.BlockCipher.Decryptor; import org.apache.parquet.format.DataPageHeader; import org.apache.parquet.format.DataPageHeaderV2; import org.apache.parquet.format.DictionaryPageHeader; @@ -48,17 +52,22 @@ public final class ParquetColumnChunkIterator private final ColumnChunkMetadata metadata; private final ChunkedInputStream input; private final OffsetIndex offsetIndex; + private final Optional decryptionContext; private long valueCount; private int dataPageCount; + private byte[] dataPageHeaderAad; + private boolean dictionaryWasRead; + public ParquetColumnChunkIterator( ParquetDataSourceId dataSourceId, Optional fileCreatedBy, ColumnDescriptor descriptor, ColumnChunkMetadata metadata, ChunkedInputStream input, - @Nullable OffsetIndex offsetIndex) + @Nullable OffsetIndex offsetIndex, + Optional decryptionContext) { this.dataSourceId = requireNonNull(dataSourceId, "dataSourceId is null"); this.fileCreatedBy = requireNonNull(fileCreatedBy, "fileCreatedBy is null"); @@ -66,6 +75,7 @@ public ParquetColumnChunkIterator( this.metadata = requireNonNull(metadata, "metadata is null"); this.input = requireNonNull(input, "input is null"); this.offsetIndex = offsetIndex; + this.decryptionContext = requireNonNull(decryptionContext, "decryptionContext is null"); } @Override @@ -80,7 +90,16 @@ public Page next() checkState(hasNext(), "No more data left to read in column (%s), metadata (%s), valueCount %s, dataPageCount %s", descriptor, metadata, valueCount, dataPageCount); try { - PageHeader pageHeader = readPageHeader(); + byte[] pageHeaderAAD = null; + if (decryptionContext.isPresent()) { + if (!dictionaryWasRead && metadata.hasDictionaryPage()) { + pageHeaderAAD = getDictionaryPageHeaderAAD(); + } + else { + pageHeaderAAD = getPageHeaderAAD(); + } + } + PageHeader pageHeader = readPageHeader(decryptionContext.map(ColumnDecryptionContext::metadataDecryptor).orElse(null), pageHeaderAAD); int uncompressedPageSize = pageHeader.getUncompressed_page_size(); int compressedPageSize = pageHeader.getCompressed_page_size(); Page result = null; @@ -90,13 +109,14 @@ public Page next() throw new ParquetCorruptionException(dataSourceId, "Column (%s) has a dictionary page after the first position in column chunk", descriptor); } result = readDictionaryPage(pageHeader, pageHeader.getUncompressed_page_size(), pageHeader.getCompressed_page_size()); + dictionaryWasRead = true; break; case DATA_PAGE: - result = readDataPageV1(pageHeader, uncompressedPageSize, compressedPageSize, getFirstRowIndex(dataPageCount, offsetIndex)); + result = readDataPageV1(pageHeader, uncompressedPageSize, compressedPageSize, getFirstRowIndex(dataPageCount, offsetIndex), dataPageCount); ++dataPageCount; break; case DATA_PAGE_V2: - result = readDataPageV2(pageHeader, uncompressedPageSize, compressedPageSize, getFirstRowIndex(dataPageCount, offsetIndex)); + result = readDataPageV2(pageHeader, uncompressedPageSize, compressedPageSize, getFirstRowIndex(dataPageCount, offsetIndex), dataPageCount); ++dataPageCount; break; default: @@ -110,10 +130,39 @@ public Page next() } } - private PageHeader readPageHeader() + private byte[] getPageHeaderAAD() + { + checkState(decryptionContext.isPresent()); + + if (dataPageHeaderAad != null) { + AesCipherUtils.quickUpdatePageAAD(dataPageHeaderAad, dataPageCount); + return dataPageHeaderAad; + } + + dataPageHeaderAad = AesCipherUtils.createModuleAAD( + decryptionContext.get().fileAad(), + ModuleType.DataPageHeader, + metadata.getRowGroupOrdinal(), + metadata.getColumnOrdinal(), + dataPageCount); + return dataPageHeaderAad; + } + + private byte[] getDictionaryPageHeaderAAD() + { + checkState(decryptionContext.isPresent()); + return AesCipherUtils.createModuleAAD( + decryptionContext.get().fileAad(), + ModuleType.DictionaryPageHeader, + metadata.getRowGroupOrdinal(), + metadata.getColumnOrdinal(), + -1); + } + + private PageHeader readPageHeader(Decryptor headerBlockDecryptor, byte[] pageHeaderAAD) throws IOException { - return Util.readPageHeader(input); + return Util.readPageHeader(input, headerBlockDecryptor, pageHeaderAAD); } private boolean hasMorePages(long valuesCountReadSoFar, int dataPageCountReadSoFar) @@ -139,7 +188,8 @@ private DataPageV1 readDataPageV1( PageHeader pageHeader, int uncompressedPageSize, int compressedPageSize, - OptionalLong firstRowIndex) + OptionalLong firstRowIndex, + int pageIndex) throws IOException { DataPageHeader dataHeaderV1 = pageHeader.getData_page_header(); @@ -151,14 +201,16 @@ private DataPageV1 readDataPageV1( firstRowIndex, getParquetEncoding(Encoding.valueOf(dataHeaderV1.getRepetition_level_encoding().name())), getParquetEncoding(Encoding.valueOf(dataHeaderV1.getDefinition_level_encoding().name())), - getParquetEncoding(Encoding.valueOf(dataHeaderV1.getEncoding().name()))); + getParquetEncoding(Encoding.valueOf(dataHeaderV1.getEncoding().name())), + pageIndex); } private DataPageV2 readDataPageV2( PageHeader pageHeader, int uncompressedPageSize, int compressedPageSize, - OptionalLong firstRowIndex) + OptionalLong firstRowIndex, + int pageIndex) throws IOException { DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); @@ -178,7 +230,8 @@ private DataPageV2 readDataPageV2( fileCreatedBy, Optional.ofNullable(dataHeaderV2.getStatistics()), descriptor.getPrimitiveType()), - dataHeaderV2.isIs_compressed()); + dataHeaderV2.isIs_compressed(), + pageIndex); } private static OptionalLong getFirstRowIndex(int pageIndex, OffsetIndex offsetIndex) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java index 00b9b937d98b..a8c6f6a004cb 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/ParquetReader.java @@ -33,6 +33,7 @@ import io.trino.parquet.ParquetWriteValidation; import io.trino.parquet.PrimitiveField; import io.trino.parquet.VariantField; +import io.trino.parquet.crypto.FileDecryptionContext; import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.parquet.metadata.PrunedBlockMetadata; import io.trino.parquet.predicate.TupleDomainParquetPredicate; @@ -150,6 +151,7 @@ public class ParquetReader private int currentPageId; private long columnIndexRowsFiltered = -1; + private final Optional decryptionContext; public ParquetReader( Optional fileCreatedBy, @@ -162,7 +164,8 @@ public ParquetReader( ParquetReaderOptions options, Function exceptionTransform, Optional parquetPredicate, - Optional writeValidation) + Optional writeValidation, + Optional decryptionContext) throws IOException { this.fileCreatedBy = requireNonNull(fileCreatedBy, "fileCreatedBy is null"); @@ -180,6 +183,7 @@ public ParquetReader( this.maxBatchSize = options.getMaxReadBlockRowCount(); this.columnReaders = new HashMap<>(); this.maxBytesPerCell = new HashMap<>(); + this.decryptionContext = requireNonNull(decryptionContext, "decryptionContext is null"); this.writeValidation = requireNonNull(writeValidation, "writeValidation is null"); validateWrite( @@ -663,7 +667,7 @@ private ColumnChunk readPrimitive(PrimitiveField field) } ChunkedInputStream columnChunkInputStream = chunkReaders.get(new ChunkKey(fieldId, currentRowGroup)); columnReader.setPageReader( - createPageReader(dataSource.getId(), columnChunkInputStream, metadata, columnDescriptor, offsetIndex, fileCreatedBy), + createPageReader(dataSource.getId(), columnChunkInputStream, metadata, columnDescriptor, offsetIndex, fileCreatedBy, decryptionContext), Optional.ofNullable(rowRanges)); } ColumnChunk columnChunk = columnReader.readPrimitive(); diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/TrinoColumnIndexStore.java b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/TrinoColumnIndexStore.java index fa9b7ae142d5..d08d28b6f857 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/reader/TrinoColumnIndexStore.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/reader/TrinoColumnIndexStore.java @@ -19,6 +19,9 @@ import io.trino.parquet.DiskRange; import io.trino.parquet.ParquetDataSource; import io.trino.parquet.ParquetReaderOptions; +import io.trino.parquet.crypto.ColumnDecryptionContext; +import io.trino.parquet.crypto.FileDecryptionContext; +import io.trino.parquet.crypto.ModuleType; import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.parquet.metadata.IndexReference; @@ -26,6 +29,7 @@ import io.trino.spi.predicate.TupleDomain; import jakarta.annotation.Nullable; import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.Util; import org.apache.parquet.hadoop.metadata.ColumnPath; import org.apache.parquet.internal.column.columnindex.ColumnIndex; @@ -47,6 +51,7 @@ import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.parquet.ParquetMetadataConverter.fromParquetColumnIndex; import static io.trino.parquet.ParquetMetadataConverter.fromParquetOffsetIndex; +import static io.trino.parquet.crypto.AesCipherUtils.createModuleAAD; import static java.util.Objects.requireNonNull; /** @@ -59,6 +64,7 @@ public class TrinoColumnIndexStore private final ParquetDataSource dataSource; private final List columnIndexReferences; private final List offsetIndexReferences; + private final Optional decryptionContext; @Nullable private Map columnIndexStore; @@ -75,9 +81,11 @@ public TrinoColumnIndexStore( ParquetDataSource dataSource, BlockMetadata block, Set columnsRead, - Set columnsFiltered) + Set columnsFiltered, + Optional decryptionContext) { this.dataSource = requireNonNull(dataSource, "dataSource is null"); + this.decryptionContext = requireNonNull(decryptionContext, "decryptionContext is null"); requireNonNull(block, "block is null"); requireNonNull(columnsRead, "columnsRead is null"); requireNonNull(columnsFiltered, "columnsFiltered is null"); @@ -90,13 +98,17 @@ public TrinoColumnIndexStore( columnIndexBuilder.add(new ColumnIndexMetadata( column.getColumnIndexReference(), path, - column.getPrimitiveType())); + column.getPrimitiveType(), + column.getRowGroupOrdinal(), + column.getColumnOrdinal())); } if (column.getOffsetIndexReference() != null && columnsRead.contains(path)) { offsetIndexBuilder.add(new ColumnIndexMetadata( column.getOffsetIndexReference(), path, - column.getPrimitiveType())); + column.getPrimitiveType(), + column.getRowGroupOrdinal(), + column.getColumnOrdinal())); } } this.columnIndexReferences = columnIndexBuilder.build(); @@ -109,14 +121,22 @@ public ColumnIndex getColumnIndex(ColumnPath column) if (columnIndexStore == null) { columnIndexStore = loadIndexes(dataSource, columnIndexReferences, (inputStream, columnMetadata) -> { try { - return fromParquetColumnIndex(columnMetadata.getPrimitiveType(), Util.readColumnIndex(inputStream)); + Optional columnContext = decryptionContext.flatMap(context -> context.getColumnDecryptionContext(columnMetadata.getPath())); + byte[] aad = columnContext + .map(context -> createModuleAAD(context.fileAad(), ModuleType.ColumnIndex, columnMetadata.getRowGroupOrdinal(), columnMetadata.getColumnOrdinal(), -1)) + .orElse(null); + BlockCipher.Decryptor thriftDecryptor = columnContext + .map(ColumnDecryptionContext::metadataDecryptor) + .orElse(null); + return fromParquetColumnIndex( + columnMetadata.getPrimitiveType(), + Util.readColumnIndex(inputStream, thriftDecryptor, aad)); } catch (IOException e) { throw new RuntimeException(e); } }); } - return columnIndexStore.get(column); } @@ -126,14 +146,20 @@ public OffsetIndex getOffsetIndex(ColumnPath column) if (offsetIndexStore == null) { offsetIndexStore = loadIndexes(dataSource, offsetIndexReferences, (inputStream, columnMetadata) -> { try { - return fromParquetOffsetIndex(Util.readOffsetIndex(inputStream)); + Optional columnContext = decryptionContext.flatMap(context -> context.getColumnDecryptionContext(columnMetadata.getPath())); + byte[] aad = columnContext + .map(context -> createModuleAAD(context.fileAad(), ModuleType.OffsetIndex, columnMetadata.getRowGroupOrdinal(), columnMetadata.getColumnOrdinal(), -1)) + .orElse(null); + BlockCipher.Decryptor thriftDecryptor = columnContext + .map(ColumnDecryptionContext::metadataDecryptor) + .orElse(null); + return fromParquetOffsetIndex(Util.readOffsetIndex(inputStream, thriftDecryptor, aad)); } catch (IOException e) { throw new RuntimeException(e); } }); } - return offsetIndexStore.get(column); } @@ -142,7 +168,8 @@ public static Optional getColumnIndexStore( BlockMetadata blockMetadata, Map, ColumnDescriptor> descriptorsByPath, TupleDomain parquetTupleDomain, - ParquetReaderOptions options) + ParquetReaderOptions options, + Optional decryptionContext) { if (!options.isUseColumnIndex() || parquetTupleDomain.isAll() || parquetTupleDomain.isNone()) { return Optional.empty(); @@ -171,7 +198,7 @@ public static Optional getColumnIndexStore( .map(column -> ColumnPath.get(column.getPath())) .collect(toImmutableSet()); - return Optional.of(new TrinoColumnIndexStore(dataSource, blockMetadata, columnsReadPaths, columnsFilteredPaths)); + return Optional.of(new TrinoColumnIndexStore(dataSource, blockMetadata, columnsReadPaths, columnsFilteredPaths, decryptionContext)); } private static Map loadIndexes( @@ -202,13 +229,17 @@ private static class ColumnIndexMetadata private final DiskRange diskRange; private final ColumnPath path; private final PrimitiveType primitiveType; + private final int rowGroupOrdinal; + private final int columnOrdinal; - private ColumnIndexMetadata(IndexReference indexReference, ColumnPath path, PrimitiveType primitiveType) + private ColumnIndexMetadata(IndexReference indexReference, ColumnPath path, PrimitiveType primitiveType, int rowGroupOrdinal, int columnOrdinal) { requireNonNull(indexReference, "indexReference is null"); this.diskRange = new DiskRange(indexReference.getOffset(), indexReference.getLength()); this.path = requireNonNull(path, "path is null"); this.primitiveType = requireNonNull(primitiveType, "primitiveType is null"); + this.rowGroupOrdinal = rowGroupOrdinal; + this.columnOrdinal = columnOrdinal; } private DiskRange getDiskRange() @@ -225,5 +256,15 @@ private PrimitiveType getPrimitiveType() { return primitiveType; } + + private int getRowGroupOrdinal() + { + return rowGroupOrdinal; + } + + private int getColumnOrdinal() + { + return columnOrdinal; + } } } diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java b/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java index 0251d897b905..5366c401d0d2 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/writer/ParquetWriter.java @@ -237,7 +237,7 @@ public void validate(ParquetDataSource input) checkState(validationBuilder.isPresent(), "validation is not enabled"); ParquetWriteValidation writeValidation = validationBuilder.get().build(); try { - ParquetMetadata parquetMetadata = MetadataReader.readFooter(input, Optional.empty(), Optional.of(writeValidation)); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(input, Optional.empty(), Optional.of(writeValidation), Optional.empty()); try (ParquetReader parquetReader = createParquetReader(input, parquetMetadata, writeValidation)) { for (SourcePage page = parquetReader.nextPage(); page != null; page = parquetReader.nextPage()) { // fully load the page @@ -294,7 +294,8 @@ private ParquetReader createParquetReader(ParquetDataSource input, ParquetMetada return new RuntimeException(exception); }, Optional.empty(), - Optional.of(writeValidation)); + Optional.of(writeValidation), + Optional.empty()); } private void recordValidation(Consumer task) diff --git a/lib/trino-parquet/src/main/java/io/trino/parquet/writer/StructColumnWriter.java b/lib/trino-parquet/src/main/java/io/trino/parquet/writer/StructColumnWriter.java index 181c1942ea35..c989a8c5d877 100644 --- a/lib/trino-parquet/src/main/java/io/trino/parquet/writer/StructColumnWriter.java +++ b/lib/trino-parquet/src/main/java/io/trino/parquet/writer/StructColumnWriter.java @@ -24,9 +24,9 @@ import java.io.IOException; import java.util.List; +import static com.google.common.base.Preconditions.checkArgument; import static io.airlift.slice.SizeOf.instanceSize; import static java.util.Objects.requireNonNull; -import static org.apache.parquet.Preconditions.checkArgument; public class StructColumnWriter implements ColumnWriter diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkColumnarFilterParquetData.java b/lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkColumnarFilterParquetData.java index b4cbdcec0b2b..09b7dd79ab9b 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkColumnarFilterParquetData.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/BenchmarkColumnarFilterParquetData.java @@ -226,7 +226,7 @@ public void setup() testData.getColumnNames(), testData.getPages()), ParquetReaderOptions.defaultOptions()); - parquetMetadata = MetadataReader.readFooter(dataSource); + parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); columnNames = columns.stream() .map(TpchColumn::getColumnName) .collect(toImmutableList()); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java b/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java index 4372aa91cda1..ebb180374394 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/ParquetTestUtils.java @@ -176,6 +176,7 @@ public static ParquetReader createParquetReader( return new RuntimeException(exception); }, Optional.of(parquetPredicate), + Optional.empty(), Optional.empty()); } diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/crypto/TestParquetEncryption.java b/lib/trino-parquet/src/test/java/io/trino/parquet/crypto/TestParquetEncryption.java new file mode 100644 index 000000000000..5997155e66d0 --- /dev/null +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/crypto/TestParquetEncryption.java @@ -0,0 +1,1038 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.parquet.crypto; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.trino.parquet.ParquetDataSource; +import io.trino.parquet.ParquetReaderOptions; +import io.trino.parquet.PrimitiveField; +import io.trino.parquet.metadata.ColumnChunkMetadata; +import io.trino.parquet.metadata.ParquetMetadata; +import io.trino.parquet.predicate.TupleDomainParquetPredicate; +import io.trino.parquet.reader.FileParquetDataSource; +import io.trino.parquet.reader.MetadataReader; +import io.trino.parquet.reader.ParquetReader; +import io.trino.parquet.reader.RowGroupInfo; +import io.trino.spi.block.Block; +import io.trino.spi.block.IntArrayBlock; +import io.trino.spi.connector.SourcePage; +import io.trino.spi.predicate.TupleDomain; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.crypto.ColumnEncryptionProperties; +import org.apache.parquet.crypto.FileEncryptionProperties; +import org.apache.parquet.crypto.ParquetCipher; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ColumnPath; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.apache.parquet.schema.PrimitiveType; +import org.apache.parquet.schema.Types; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.Supplier; + +import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; +import static io.trino.parquet.predicate.PredicateUtils.getFilteredRowGroups; +import static io.trino.spi.predicate.Domain.singleValue; +import static io.trino.spi.type.IntegerType.INTEGER; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.Files.createTempFile; +import static org.apache.parquet.hadoop.ParquetFileWriter.Mode.OVERWRITE; +import static org.apache.parquet.hadoop.metadata.CompressionCodecName.SNAPPY; +import static org.apache.parquet.hadoop.metadata.CompressionCodecName.UNCOMPRESSED; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.joda.time.DateTimeZone.UTC; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; + +/** + * Unit‑tests exercising Trino’s PME path. + */ +// ExampleParquetWriter is not thread-safe +@TestInstance(PER_CLASS) +@Execution(SAME_THREAD) +public final class TestParquetEncryption +{ + private static final ColumnPath AGE_PATH = ColumnPath.fromDotString("age"); + private static final ColumnPath ID_PATH = ColumnPath.fromDotString("id"); + + private static final byte[] KEY_AGE = "colKeyIs16ByteA?".getBytes(UTF_8); + private static final byte[] KEY_ID = "colKeyIs16ByteB?".getBytes(UTF_8); + private static final byte[] KEY_FOOT = "footKeyIs16Byte?".getBytes(UTF_8); + + // one‑column schema + private static final MessageType AGE_SCHEMA = + MessageTypeParser.parseMessageType("message doc { required int32 age; }"); + + // two‑column schema + private static final MessageType TWO_COL_SCHEMA = MessageTypeParser.parseMessageType( + "message doc { required int32 age; required int32 id; }"); + + /** + * Column encryption only – footer left in plaintext. + */ + @ParameterizedTest(name = "checkFooterIntegrity={0}, compressed={1}") + @CsvSource({ + "false,false", + "false,true", + "true,false", + "true,true" + }) + void columnOnlyFooterPlaintext(boolean checkFooterIntegrity, boolean compressed) + throws IOException + { + File file = createTempFile("pme‑col‑only", ".parquet").toFile(); + file.deleteOnExit(); + + writeSingleColumnFile(file, compressed, /*encryptFooter*/ false); + + List values = readSingleColumnFile( + file, + new TestingKeyRetriever(checkFooterIntegrity ? Optional.of(KEY_FOOT) : Optional.empty(), Optional.of(KEY_AGE), Optional.empty()), + checkFooterIntegrity, + Optional.empty()); + + verifySequence(values, 100); + } + + /** + * Column+ footer encryption (same column as above). + */ + @ParameterizedTest(name = "compressed={0}") + @CsvSource({"false", "true"}) + void columnAndFooter(boolean compressed) + throws IOException + { + File file = createTempFile("pme‑col‑foot", ".parquet").toFile(); + file.deleteOnExit(); + + writeSingleColumnFile(file, compressed, /*encryptFooter*/ true); + + List values = readSingleColumnFile( + file, + new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty()), + /*checkFooterIntegrity*/ true, + Optional.empty()); + + verifySequence(values, 100); + } + + /** + * Single column encrypted with footer column. + */ + @ParameterizedTest(name = "encryptFooter={0}") + @CsvSource({"false", "true"}) + void columnEncryptedWithFooterKey(boolean encryptFooter) + throws IOException + { + File file = createTempFile("pme‑footerKey‑col", ".parquet").toFile(); + file.deleteOnExit(); + + // write: column encrypted with footer key, footer encrypted as usual + writeSingleColumnFile(file, /*compressed*/ false, encryptFooter, /*columnEncryptedWithFooterKey*/ true, Optional.empty(), () -> range(0, 100), OptionalInt.empty(), OptionalInt.empty()); + + // read: supply ONLY the footer key (no column key necessary) + List values = readSingleColumnFile( + file, + new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.empty(), Optional.empty()), + /*checkFooterIntegrity*/ true, + Optional.empty()); + + verifySequence(values, 100); // data round‑trip OK + } + + @ParameterizedTest(name = "supplyPrefix={0}") + @CsvSource({"true", "false"}) + void aadPrefixRoundTrip(boolean supplyPrefix) + throws IOException + { + File file = createTempFile("pme‑aad", ".parquet").toFile(); + file.deleteOnExit(); + + byte[] prefix = "fileAAD‑prefix".getBytes(UTF_8); + writeSingleColumnFile(file, /*compressed*/ true, /*encryptFooter*/ true, /*columnEncryptedWithFooterKey*/ false, Optional.of(prefix), () -> range(0, 100), OptionalInt.empty(), OptionalInt.empty()); + + // build FileDecryptionProperties with / without the required prefix + FileDecryptionProperties.Builder properties = FileDecryptionProperties.builder() + .withKeyRetriever(new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty())); + + if (supplyPrefix) { + properties.withAadPrefix(prefix); + // correct prefix => read succeeds + List values = readSingleColumnFile( + file, + new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty()), + true, // footer integrity + Optional.of(prefix)); // pass prefix + + verifySequence(values, 100); + } + else { + // missing / wrong prefix => should fail while reading footer + assertThatThrownBy(() -> readSingleColumnFile( + file, + new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty()), + true, + Optional.empty())) // NO prefix + .isInstanceOfAny(ParquetCryptoException.class); + } + } + + @ParameterizedTest(name = "rows={0}, rowGroupSize={1}, pageSize={2}, encryptFooter={3}, compressed={4}") + @CsvSource({ + // rows, rowGroupSizeBytes, pageSizeBytes, encryptFooter, compressed + "2000, 1024, 128, true, false", + "1500, 2048, 256, false, true", + }) + void multipleRowGroups(int rows, int rowGroupSize, int pageSize, boolean encryptFooter, boolean compressed) + throws IOException + { + File file = createTempFile("pme-multirg", ".parquet").toFile(); + file.deleteOnExit(); + + writeSingleColumnFile( + file, + compressed, + encryptFooter, + /* columnEncryptedWithFooterKey */ false, + Optional.empty(), + () -> range(0, rows), + OptionalInt.of(rowGroupSize), + OptionalInt.of(pageSize)); + + // sanity: we really created >1 row group + try (ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build())) { + FileDecryptionProperties properties = FileDecryptionProperties.builder() + .withKeyRetriever(new TestingKeyRetriever( + /* footer */ encryptFooter ? Optional.of(KEY_FOOT) : Optional.empty(), + /* column */ Optional.of(KEY_AGE), + Optional.empty())) + .withCheckFooterIntegrity(encryptFooter) + .build(); + ParquetMetadata metadata = MetadataReader.readFooter(source, Optional.empty(), Optional.empty(), Optional.of(properties)); + assertThat(metadata.getBlocks().size()).isGreaterThan(1); + } + + List values = readSingleColumnFile( + file, + new TestingKeyRetriever(encryptFooter ? Optional.of(KEY_FOOT) : Optional.empty(), Optional.of(KEY_AGE), Optional.empty()), + encryptFooter, + Optional.empty()); + + verifySequence(values, rows); + } + + /** + * Two columns; footer encrypted; one column plaintext, other encrypted. + */ + @Test + void mixedEncryptedAndPlaintextColumns() + throws IOException + { + File file = createTempFile("pme‑mixed‑cols", ".parquet").toFile(); + file.deleteOnExit(); + + // age column encrypted, id column plaintext + writeTwoColumnFile(file, /*encryptAge*/ true, /*encryptId*/ false); + + // Provide keys for both footer & encrypted age column + Map> data = readTwoColumnFile(file, new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty())); + + verifySequence(data.get("age"), 100); + verifyReverseSequence(data.get("id"), 100); + } + + /** + * Two columns; different keys; refuse access to one column. + */ + @Test + void oneColumnAccessibleOtherInaccessible() + throws IOException + { + File file = createTempFile("pme‑locked‑col", ".parquet").toFile(); + file.deleteOnExit(); + + // both columns encrypted with different keys + writeTwoColumnFile(file, /*encryptAge*/ true, /*encryptId*/ true); + + // reader has footer key + KEY_AGE only, not KEY_ID + TestingKeyRetriever retriever = new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty()); + + List values = readSingleColumnFile(file, retriever, true, Optional.empty()); + + verifySequence(values, 100); + } + + /** + * Single column encrypted, written with dictionary encoding. + */ + @Test + void dictionaryEncodedEncryptedColumn() + throws IOException + { + File file = createTempFile("pme‑dict", ".parquet").toFile(); + file.deleteOnExit(); + + // create a very small domain so writer chooses dictionary + writeSingleColumnFile( + file, + /*compressed*/ false, + /*encryptFooter*/ true, + /*columnEncryptedWithFooterKey*/ false, + Optional.empty(), + () -> List.of(1, 2, 3, 1, 2, 3), + OptionalInt.empty(), + OptionalInt.empty()); // limited distincts + + List values = readSingleColumnFile( + file, + new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty()), + true, + Optional.empty()); + + assertThat(new HashSet<>(values)).containsExactlyInAnyOrder(1, 2, 3); + + // ── extra assertion: verify a dictionary really exists ── + try (ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build())) { + FileDecryptionProperties properties = FileDecryptionProperties.builder() + .withKeyRetriever(new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(KEY_AGE), Optional.empty())) + .build(); + + ParquetMetadata metadata = MetadataReader.readFooter(source, Optional.empty(), Optional.empty(), Optional.of(properties)); + + // first (and only) row‑group → column‑chunk for "age" + ColumnChunkMetadata chunk = + metadata.getBlocks().getFirst().columns().stream() + .filter(column -> column.getPath().equals(AGE_PATH)) + .findFirst() + .orElseThrow(); + + // 1) dictionary page must be present + assertThat(chunk.getDictionaryPageOffset()).isGreaterThan(0); + + // 2) PLAIN_DICTIONARY (or RLE_DICTIONARY in v2) must be in the encoding list + assertThat(chunk.getEncodings()).anyMatch(Encoding::usesDictionary); + + // 3) (optional) Trino helper: all data pages are dictionary‑encoded + assertThat(io.trino.parquet.ParquetReaderUtils.isOnlyDictionaryEncodingPages(chunk)).isTrue(); + } + } + + @Test + void readFailsWithoutFooterKey() + throws IOException + { + File file = createTempFile("pme-no-footer-key", ".parquet").toFile(); + file.deleteOnExit(); + + // footer + column encrypted + writeSingleColumnFile(file, /*compressed*/ false, /*encryptFooter*/ true); + + assertThatThrownBy(() -> readSingleColumnFile( + file, + new TestingKeyRetriever(/* footer */ Optional.empty(), + /* column */ Optional.of(KEY_AGE), + Optional.empty()), + /*checkFooterIntegrity*/ true, + Optional.empty())) + .isInstanceOf(ParquetCryptoException.class); + } + + /** + * Footer key is present, but the column key is missing. + * Footer decrypts; column decryption must fail. + */ + @Test + void readFailsWithoutColumnKey() + throws IOException + { + File file = createTempFile("pme-no-column-key", ".parquet").toFile(); + file.deleteOnExit(); + + // footer + column encrypted + writeSingleColumnFile(file, /*compressed*/ false, /*encryptFooter*/ true); + + assertThatThrownBy(() -> readSingleColumnFile( + file, + new TestingKeyRetriever(/* footer */ Optional.of(KEY_FOOT), + /* column */ Optional.empty(), + Optional.empty()), + /*checkFooterIntegrity*/ true, + Optional.empty())) + .isInstanceOf(ParquetCryptoException.class); + } + + @Test + void readFailsWithInvalidFooterKey() + throws IOException + { + File file = createTempFile("pme-bad-footer", ".parquet").toFile(); + file.deleteOnExit(); + + writeSingleColumnFile(file, /*compressed*/ false, /*encryptFooter*/ true); + + byte[] wrongFooter = "thisIsTheBadKey!".getBytes(UTF_8); // 16 bytes != real key + + assertThatThrownBy(() -> readSingleColumnFile( + file, + new TestingKeyRetriever(Optional.of(wrongFooter), Optional.of(KEY_AGE), Optional.empty()), + /*checkFooterIntegrity*/ true, + Optional.empty())) + .isInstanceOf(ParquetCryptoException.class); + } + + /** + * Footer key is correct but the column key is wrong. + * Footer decrypts, column decryption must still fail. + */ + @Test + void readFailsWithInvalidColumnKey() + throws IOException + { + File file = createTempFile("pme-bad-column", ".parquet").toFile(); + file.deleteOnExit(); + + writeSingleColumnFile(file, /*compressed*/ false, /*encryptFooter*/ true); + + byte[] wrongColumn = "badColumnKey123".getBytes(UTF_8); // 16 bytes + + assertThatThrownBy(() -> readSingleColumnFile( + file, + new TestingKeyRetriever(Optional.of(KEY_FOOT), Optional.of(wrongColumn), Optional.empty()), + /*checkFooterIntegrity*/ true, + Optional.empty())) + .isInstanceOf(ParquetCryptoException.class); + } + + @Test + void encryptedDictionaryPruningTwoColumns() + throws IOException + { + File file = createTempFile("pme-dict-2cols", ".parquet").toFile(); + file.deleteOnExit(); + + int missingAge = 7; + int missingId = 3; + writeTwoColumnEncryptedDictionaryFile(file, missingAge, missingId); + + // Open with footer key + AGE key only (no ID key) + try (ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build())) { + FileDecryptionProperties props = FileDecryptionProperties.builder() + .withKeyRetriever(new TestingKeyRetriever( + Optional.of(KEY_FOOT), // footer + Optional.of(KEY_AGE), // age column key + Optional.empty())) // NO id key + .withCheckFooterIntegrity(true) + .build(); + + ParquetMetadata metadata = MetadataReader.readFooter( + source, Optional.empty(), Optional.empty(), Optional.of(props)); + + ColumnDescriptor age = new ColumnDescriptor( + new String[] {"age"}, + Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("age"), + 0, 0); + + ColumnDescriptor id = new ColumnDescriptor( + new String[] {"id"}, + Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("id"), + 0, 0); + + // ——— Predicate on accessible column (age = missingAge) → dictionary-based pruning to 0 ——— + TupleDomain domainAge = TupleDomain.withColumnDomains(ImmutableMap.of(age, singleValue(INTEGER, (long) missingAge))); + + TupleDomainParquetPredicate predicateAge = new TupleDomainParquetPredicate( + domainAge, ImmutableList.of(age), UTC); + + List groupsAge = getFilteredRowGroups( + 0, + source.getEstimatedSize(), + source, + metadata, + List.of(domainAge), + List.of(predicateAge), + ImmutableMap.of(ImmutableList.of("age"), age), + UTC, + 200, + ParquetReaderOptions.builder().build()); + + // No row-groups should pass after dictionary pruning + assertThat(groupsAge).isEmpty(); + + // ——— Predicate on inaccessible column (id = missingId) → should fail (no column key) ——— + TupleDomain domainId = TupleDomain.withColumnDomains(ImmutableMap.of(id, singleValue(INTEGER, (long) missingId))); + + TupleDomainParquetPredicate predicateId = new TupleDomainParquetPredicate(domainId, ImmutableList.of(id), UTC); + + assertThatThrownBy(() -> getFilteredRowGroups( + 0, + source.getEstimatedSize(), + source, + metadata, + List.of(domainId), + List.of(predicateId), + ImmutableMap.of(ImmutableList.of("id"), id), + UTC, + 200, + ParquetReaderOptions.builder().build())) + // keep this broad; different layers may surface different messages + .hasMessageMatching("(?s).*access.*column.*id.*|.*decrypt.*id.*|.*key.*id.*"); + } + } + + @Test + void encryptedBloomFilterPruningTwoColumns() + throws IOException + { + File file = createTempFile("pme-bloom-2cols", ".parquet").toFile(); + file.deleteOnExit(); + + int missingAge = 7; + int missingId = 3; + writeTwoColumnEncryptedBloomFile(file, missingAge, missingId); + assertBloomFiltersPresent(file); + + try (ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build())) { + // Footer + AGE key only (no ID key) + FileDecryptionProperties props = FileDecryptionProperties.builder() + .withKeyRetriever(new TestingKeyRetriever( + Optional.of(KEY_FOOT), // footer + Optional.of(KEY_AGE), // age column key + Optional.empty())) // NO id key + .withCheckFooterIntegrity(true) + .build(); + + ParquetMetadata metadata = MetadataReader.readFooter(source, Optional.empty(), Optional.empty(), Optional.of(props)); + + ColumnDescriptor age = new ColumnDescriptor( + new String[] {"age"}, + Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("age"), + 0, 0); + + ColumnDescriptor id = new ColumnDescriptor( + new String[] {"id"}, + Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("id"), + 0, 0); + + // --- Predicate on accessible column (age == missingAge) → Bloom filter should prune to 0 + TupleDomain domainAge = TupleDomain.withColumnDomains( + ImmutableMap.of(age, singleValue(INTEGER, (long) missingAge))); + TupleDomainParquetPredicate predicateAge = new TupleDomainParquetPredicate(domainAge, ImmutableList.of(age), UTC); + + List groupsAge = getFilteredRowGroups( + 0, + source.getEstimatedSize(), + source, + metadata, + List.of(domainAge), + List.of(predicateAge), + ImmutableMap.of(ImmutableList.of("age"), age), + UTC, + 200, + ParquetReaderOptions.builder().build()); + + assertThat(groupsAge).isEmpty(); // pruned by encrypted Bloom filter + + // Sanity: present value should not prune to 0 + TupleDomain domainAgePresent = TupleDomain.withColumnDomains( + ImmutableMap.of(age, singleValue(INTEGER, 5L))); + TupleDomainParquetPredicate predicateAgePresent = new TupleDomainParquetPredicate(domainAgePresent, ImmutableList.of(age), UTC); + List groupsAgePresent = getFilteredRowGroups( + 0, + source.getEstimatedSize(), + source, + metadata, + List.of(domainAgePresent), + List.of(predicateAgePresent), + ImmutableMap.of(ImmutableList.of("age"), age), + UTC, + 200, + ParquetReaderOptions.builder().build()); + assertThat(groupsAgePresent).isNotEmpty(); + + // --- Predicate on inaccessible column (id == missingId) → should fail (no column key) + TupleDomain domainId = TupleDomain.withColumnDomains( + ImmutableMap.of(id, singleValue(INTEGER, (long) missingId))); + TupleDomainParquetPredicate predicateId = new TupleDomainParquetPredicate(domainId, ImmutableList.of(id), UTC); + + assertThatThrownBy(() -> getFilteredRowGroups( + 0, + source.getEstimatedSize(), + source, + metadata, + List.of(domainId), + List.of(predicateId), + ImmutableMap.of(ImmutableList.of("id"), id), + UTC, + 200, + ParquetReaderOptions.builder().build())) + .hasMessageMatching("(?s).*access.*column.*id.*|.*decrypt.*id.*|.*key.*id.*"); + } + } + + /** + * Single‑column writer with knobs. + */ + private static void writeSingleColumnFile(File target, boolean compressed, boolean encryptFooter) + throws IOException + { + writeSingleColumnFile(target, compressed, encryptFooter, false, Optional.empty(), () -> range(0, 100), OptionalInt.empty(), OptionalInt.empty()); + } + + /** + * Overload that takes explicit values supplier (for dictionary case). + */ + private static void writeSingleColumnFile( + File target, + boolean compressed, + boolean encryptFooter, + boolean columnEncryptedWithFooterKey, + Optional aadPrefix, + Supplier> valuesSupplier, + OptionalInt rowGroupSizeBytes, + OptionalInt pageSizeBytes) + throws IOException + { + ColumnEncryptionProperties.Builder ageEncryptionBuilder = ColumnEncryptionProperties.builder(AGE_PATH); + if (!columnEncryptedWithFooterKey) { + ageEncryptionBuilder.withKey(KEY_AGE); + } + ColumnEncryptionProperties ageEncryption = ageEncryptionBuilder.build(); + + FileEncryptionProperties.Builder fileEncryption = FileEncryptionProperties.builder(KEY_FOOT) + .withAlgorithm(ParquetCipher.AES_GCM_CTR_V1) + .withEncryptedColumns(ImmutableMap.of(AGE_PATH, ageEncryption)); + + aadPrefix.ifPresent(prefix -> { + fileEncryption.withAADPrefix(prefix); + fileEncryption.withoutAADPrefixStorage(); + }); + + if (!encryptFooter) { + fileEncryption.withPlaintextFooter(); + } + + ExampleParquetWriter.Builder builder = ExampleParquetWriter + .builder(new Path(target.getAbsolutePath())) + .withType(AGE_SCHEMA) + .withConf(new Configuration()) + .withEncryption(fileEncryption.build()) + .withWriteMode(OVERWRITE) + .withCompressionCodec(compressed ? SNAPPY : UNCOMPRESSED); + + rowGroupSizeBytes.ifPresent(builder::withRowGroupSize); + // tiny page => even 100 ints give us ≥ 2 data pages + builder.withPageSize(pageSizeBytes.orElse(64)); + + try (ParquetWriter writer = builder.build()) { + SimpleGroupFactory factory = new SimpleGroupFactory(AGE_SCHEMA); + for (int value : valuesSupplier.get()) { + writer.write(factory.newGroup().append("age", value)); + } + } + } + + /** + * Two‑column writer. + */ + private static void writeTwoColumnFile( + File target, + boolean encryptAge, + boolean encryptId) + throws IOException + { + ImmutableMap.Builder columnMap = ImmutableMap.builder(); + if (encryptAge) { + columnMap.put(AGE_PATH, ColumnEncryptionProperties.builder(AGE_PATH).withKey(KEY_AGE).build()); + } + if (encryptId) { + columnMap.put(ID_PATH, ColumnEncryptionProperties.builder(ID_PATH).withKey(KEY_ID).build()); + } + + FileEncryptionProperties fileEncryption = FileEncryptionProperties.builder(KEY_FOOT) + .withAlgorithm(ParquetCipher.AES_GCM_CTR_V1) + .withEncryptedColumns(columnMap.buildOrThrow()) + .build(); + + ExampleParquetWriter.Builder builder = ExampleParquetWriter.builder(new Path(target.getAbsolutePath())) + .withType(TWO_COL_SCHEMA) + .withConf(new Configuration()) + .withEncryption(fileEncryption) + .withWriteMode(OVERWRITE) + // tiny page => even 100 ints give us ≥ 2 data pages + .withPageSize(64); + + try (ParquetWriter writer = builder.build()) { + SimpleGroupFactory factory = new SimpleGroupFactory(TWO_COL_SCHEMA); + for (int i = 0; i < 100; i++) { + writer.write(factory.newGroup().append("id", 100 - i).append("age", i)); + } + } + } + + private static void writeTwoColumnEncryptedDictionaryFile(File target, int missingAge, int missingId) + throws IOException + { + FileEncryptionProperties fileEncryption = FileEncryptionProperties.builder(KEY_FOOT) + .withAlgorithm(ParquetCipher.AES_GCM_CTR_V1) + .withEncryptedColumns(ImmutableMap.of( + AGE_PATH, ColumnEncryptionProperties.builder(AGE_PATH).withKey(KEY_AGE).build(), + ID_PATH, ColumnEncryptionProperties.builder(ID_PATH).withKey(KEY_ID).build())) + .build(); + + ExampleParquetWriter.Builder builder = ExampleParquetWriter.builder(new Path(target.getAbsolutePath())) + .withType(TWO_COL_SCHEMA) + .withConf(new Configuration()) + .withEncryption(fileEncryption) + .withWriteMode(OVERWRITE) + .withPageSize(1024); // small pages → strong chance of dictionary encoding + + writeSampleData(builder, missingAge, missingId); + assertDictionariesPresent(target); + } + + private static void assertDictionariesPresent(File file) + throws IOException + { + try (ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build())) { + FileDecryptionProperties properties = FileDecryptionProperties.builder() + // for metadata inspection we provide BOTH column keys + footer key + .withKeyRetriever(new TestingKeyRetriever( + Optional.of(KEY_FOOT), + Optional.of(KEY_AGE), + Optional.of(KEY_ID))) + .withCheckFooterIntegrity(true) + .build(); + + ParquetMetadata metadata = MetadataReader.readFooter(source, Optional.empty(), Optional.empty(), Optional.of(properties)); + + ColumnChunkMetadata age = metadata.getBlocks().getFirst().columns().stream() + .filter(context -> context.getPath().equals(AGE_PATH)) + .findFirst().orElseThrow(); + + ColumnChunkMetadata id = metadata.getBlocks().getFirst().columns().stream() + .filter(context -> context.getPath().equals(ID_PATH)) + .findFirst().orElseThrow(); + + // dictionary page present + assertThat(age.getDictionaryPageOffset()).isGreaterThan(0); + assertThat(id.getDictionaryPageOffset()).isGreaterThan(0); + + // column encodings include a dictionary encoding + assertThat(age.getEncodings()).anyMatch(org.apache.parquet.column.Encoding::usesDictionary); + assertThat(id.getEncodings()).anyMatch(org.apache.parquet.column.Encoding::usesDictionary); + + // (optional) every data page is dictionary-encoded + assertThat(io.trino.parquet.ParquetReaderUtils.isOnlyDictionaryEncodingPages(age)).isTrue(); + assertThat(io.trino.parquet.ParquetReaderUtils.isOnlyDictionaryEncodingPages(id)).isTrue(); + } + } + + private static void writeTwoColumnEncryptedBloomFile(File target, int missingAge, int missingId) + throws IOException + { + // Encryption: both columns with different keys; encrypted footer + FileEncryptionProperties fileEncryption = FileEncryptionProperties.builder(KEY_FOOT) + .withAlgorithm(ParquetCipher.AES_GCM_CTR_V1) + .withEncryptedColumns(ImmutableMap.of( + AGE_PATH, ColumnEncryptionProperties.builder(AGE_PATH).withKey(KEY_AGE).build(), + ID_PATH, ColumnEncryptionProperties.builder(ID_PATH).withKey(KEY_ID).build())) + .build(); + + // Enable bloom filters and select columns + ExampleParquetWriter.Builder builder = ExampleParquetWriter.builder(new Path(target.getAbsolutePath())) + .withType(TWO_COL_SCHEMA) + .withEncryption(fileEncryption) + .withWriteMode(OVERWRITE) + .withPageSize(1024) + // Bloom filters won't be used with dictionary encoding + .withDictionaryEncoding(false) + .withBloomFilterEnabled(true); + + writeSampleData(builder, missingAge, missingId); + assertBloomFiltersPresent(target); + } + + private static void writeSampleData(ExampleParquetWriter.Builder builder, int missingAge, int missingId) + throws IOException + { + try (ParquetWriter writer = builder.build()) { + SimpleGroupFactory factory = new SimpleGroupFactory(TWO_COL_SCHEMA); + for (int i = 0; i < 5000; i++) { + int id = i % 11; + // ensure 'missingId' not present + if (id == missingId) { + id = (id + 1) % 11; + } + int age = i % 11; + // ensure 'missingAge' not present + if (age == missingAge) { + age = (age + 1) % 11; + } + writer.write(factory.newGroup().append("id", id).append("age", age)); + } + } + } + + private static void assertBloomFiltersPresent(File file) + throws IOException + { + try (ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build())) { + // Provide ALL keys for metadata inspection + FileDecryptionProperties properties = FileDecryptionProperties.builder() + .withKeyRetriever(new TestingKeyRetriever( + Optional.of(KEY_FOOT), + Optional.of(KEY_AGE), + Optional.of(KEY_ID))) + .withCheckFooterIntegrity(true) + .build(); + + ParquetMetadata metadata = MetadataReader.readFooter(source, Optional.empty(), Optional.empty(), Optional.of(properties)); + + ColumnChunkMetadata age = metadata.getBlocks().getFirst().columns().stream() + .filter(context -> context.getPath().equals(AGE_PATH)) + .findFirst().orElseThrow(); + + ColumnChunkMetadata id = metadata.getBlocks().getFirst().columns().stream() + .filter(context -> context.getPath().equals(ID_PATH)) + .findFirst().orElseThrow(); + + // Bloom filter offsets must be present for both columns + assertThat(age.getBloomFilterOffset()).isGreaterThan(0L); + assertThat(id.getBloomFilterOffset()).isGreaterThan(0L); + } + } + + /** + * Reads the single‑column file and returns the “age” values. + */ + private static List readSingleColumnFile( + File file, + DecryptionKeyRetriever keyRetriever, + boolean checkFooterIntegrity, + Optional aadPrefix) + throws IOException + { + ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build()); + + FileDecryptionProperties.Builder properties = FileDecryptionProperties + .builder() + .withKeyRetriever(keyRetriever) + .withCheckFooterIntegrity(checkFooterIntegrity); + aadPrefix.ifPresent(properties::withAadPrefix); + + ParquetMetadata metadata = MetadataReader.readFooter( + source, Optional.empty(), Optional.empty(), Optional.of(properties.build())); + + ColumnDescriptor descriptor = new ColumnDescriptor( + new String[] {"age"}, + Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("age"), + 0, 0); + + Map, ColumnDescriptor> byPath = ImmutableMap.of( + ImmutableList.of("age"), descriptor); + + TupleDomain domain = TupleDomain.all(); + TupleDomainParquetPredicate predicate = new TupleDomainParquetPredicate( + domain, ImmutableList.of(descriptor), UTC); + + List groups = getFilteredRowGroups( + 0, source.getEstimatedSize(), + source, metadata, + List.of(domain), List.of(predicate), + byPath, UTC, 200, ParquetReaderOptions.builder().build()); + + PrimitiveField field = new PrimitiveField(INTEGER, true, descriptor, 0); + io.trino.parquet.Column column = new io.trino.parquet.Column("age", field); + + try (ParquetReader reader = new ParquetReader( + Optional.ofNullable(metadata.getFileMetaData().getCreatedBy()), + List.of(column), + false, + groups, + source, + UTC, + newSimpleAggregatedMemoryContext(), + ParquetReaderOptions.builder().build(), + RuntimeException::new, + Optional.of(predicate), + Optional.empty(), + metadata.getDecryptionContext())) { + List out = new ArrayList<>(); + SourcePage page; + while ((page = reader.nextPage()) != null) { + Block block = page.getBlock(0); + IntArrayBlock ints = (IntArrayBlock) block.getUnderlyingValueBlock(); + for (int i = 0; i < ints.getPositionCount(); i++) { + out.add(block.isNull(i) ? null : ints.getInt(i)); + } + } + return out; + } + finally { + source.close(); + } + } + + /** + * Reads both columns and returns a map “age” → values, “id → values. + */ + private static Map> readTwoColumnFile( + File file, DecryptionKeyRetriever retriever) + throws IOException + { + ParquetDataSource source = new FileParquetDataSource(file, ParquetReaderOptions.builder().build()); + + FileDecryptionProperties properties = FileDecryptionProperties + .builder().withKeyRetriever(retriever).withCheckFooterIntegrity(true).build(); + + ParquetMetadata metadata = MetadataReader.readFooter(source, Optional.empty(), Optional.empty(), Optional.of(properties)); + + ColumnDescriptor ageDescriptor = new ColumnDescriptor( + new String[] {"age"}, + Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("age"), 0, 0); + + ColumnDescriptor idDescriptor = new ColumnDescriptor( + new String[] {"id"}, + Types.required(PrimitiveType.PrimitiveTypeName.INT32).named("id"), 0, 0); + + Map, ColumnDescriptor> byPath = ImmutableMap.of( + ImmutableList.of("age"), ageDescriptor, + ImmutableList.of("id"), idDescriptor); + + TupleDomainParquetPredicate predicate = new TupleDomainParquetPredicate( + TupleDomain.all(), ImmutableList.of(ageDescriptor, idDescriptor), UTC); + + List groups = getFilteredRowGroups( + 0, source.getEstimatedSize(), source, metadata, + List.of(TupleDomain.all()), List.of(predicate), + byPath, UTC, 200, ParquetReaderOptions.builder().build()); + + PrimitiveField ageField = new PrimitiveField(INTEGER, true, ageDescriptor, 0); + PrimitiveField idField = new PrimitiveField(INTEGER, true, idDescriptor, 1); + + List columns = List.of( + new io.trino.parquet.Column("age", ageField), + new io.trino.parquet.Column("id", idField)); + + try (ParquetReader reader = new ParquetReader( + Optional.ofNullable(metadata.getFileMetaData().getCreatedBy()), + columns, + false, groups, source, + UTC, + newSimpleAggregatedMemoryContext(), + ParquetReaderOptions.builder().build(), + RuntimeException::new, + Optional.of(predicate), + Optional.empty(), + metadata.getDecryptionContext())) { + List ages = new ArrayList<>(); + List ids = new ArrayList<>(); + + SourcePage page; + while ((page = reader.nextPage()) != null) { + for (int column = 0; column < 2; column++) { + Block block = page.getBlock(column); + IntArrayBlock ints = (IntArrayBlock) block.getUnderlyingValueBlock(); + List values = (column == 0) ? ages : ids; + for (int i = 0; i < ints.getPositionCount(); i++) { + values.add(block.isNull(i) ? null : ints.getInt(i)); + } + } + } + return Map.of("age", ages, "id", ids); + } + finally { + source.close(); + } + } + + private static void verifySequence(List actual, int n) + { + assertThat(actual).hasSize(n); + for (int i = 0; i < n; i++) { + assertThat(actual.get(i)).isEqualTo(i); + } + } + + private static void verifyReverseSequence(List actual, int n) + { + assertThat(actual).hasSize(n); + for (int i = 0; i < n; i++) { + assertThat(actual.get(i)).isEqualTo(100 - i); + } + } + + private static List range(int fromInclusive, int toExclusive) + { + List out = new ArrayList<>(toExclusive - fromInclusive); + for (int i = fromInclusive; i < toExclusive; i++) { + out.add(i); + } + return out; + } + + private static final class TestingKeyRetriever + implements DecryptionKeyRetriever + { + private final Optional footerKey; + private final Optional ageColumnKey; + private final Optional idColumnKey; + + public TestingKeyRetriever(Optional footerKey, Optional ageColumnKey, Optional idColumnKey) + { + this.footerKey = footerKey; + this.ageColumnKey = ageColumnKey; + this.idColumnKey = idColumnKey; + } + + @Override + public Optional getColumnKey(ColumnPath path, Optional keyMetadata) + { + if ("age".equals(path.toDotString())) { + return ageColumnKey; + } + if ("id".equals(path.toDotString())) { + return idColumnKey; + } + return Optional.empty(); + } + + @Override + public Optional getFooterKey(Optional keyMetadata) + { + return footerKey; + } + } +} diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderBenchmark.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderBenchmark.java index 84e0abc7631d..7c9d414a41a3 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderBenchmark.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderBenchmark.java @@ -105,7 +105,7 @@ public int read() throws IOException { ColumnReader columnReader = columnReaderFactory.create(field, newSimpleAggregatedMemoryContext()); - PageReader pageReader = new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, dataPages.iterator(), false, false); + PageReader pageReader = new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, dataPages.iterator(), false, false, Optional.empty(), -1, -1); columnReader.setPageReader(pageReader, Optional.empty()); int rowsRead = 0; while (rowsRead < dataPositions) { @@ -133,7 +133,8 @@ private DataPage createDataPage(ValuesWriter writer, int valuesCount) OptionalLong.empty(), RLE, RLE, - getParquetEncoding(writer.getEncoding())); + getParquetEncoding(writer.getEncoding()), + 0); } protected static void run(Class clazz) diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderRowRangesTest.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderRowRangesTest.java index 6a3fccb1e281..e7af59113053 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderRowRangesTest.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderRowRangesTest.java @@ -285,8 +285,8 @@ public String toString() private static ColumnReaderInput[] getColumnReaderInputs(ColumnReaderProvider columnReaderProvider) { Object[][] definitionLevelsProviders = Arrays.stream(DefinitionLevelsProvider.ofDefinitionLevel( - columnReaderProvider.getField().getDefinitionLevel(), - columnReaderProvider.getField().isRequired())) + columnReaderProvider.getField().getDefinitionLevel(), + columnReaderProvider.getField().isRequired())) .collect(toDataProvider()); PrimitiveField field = columnReaderProvider.getField(); @@ -564,7 +564,10 @@ else if (dictionaryEncoding == DictionaryEncoding.MIXED) { UNCOMPRESSED, inputPages.iterator(), dictionaryEncoding == DictionaryEncoding.ALL || (dictionaryEncoding == DictionaryEncoding.MIXED && testingPages.size() == 1), - false); + false, + Optional.empty(), + -1, + -1); } private static List createDataPages(List testingPages, ValuesWriter encoder, int maxDef, boolean required) @@ -599,7 +602,8 @@ private static DataPage createDataPage(TestingPage testingPage, ValuesWriter enc valueCount * 4, OptionalLong.of(testingPage.pageRowRange().start()), null, - false); + false, + 0); encoder.reset(); return dataPage; } diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java index 8a51ff5f995c..4ff78c4f1a74 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/AbstractColumnReaderTest.java @@ -660,7 +660,8 @@ protected static DataPage createDataPage( OptionalLong.empty(), getParquetEncoding(repetitionWriter.getEncoding()), getParquetEncoding(definitionWriter.getEncoding()), - encoding); + encoding, + 0); } return new DataPageV2( valueCount, @@ -673,7 +674,8 @@ protected static DataPage createDataPage( definitionBytes.length + repetitionBytes.length + valueBytes.length, OptionalLong.empty(), null, - false); + false, + 0); } protected static PageReader getPageReaderMock(List dataPages, @Nullable DictionaryPage dictionaryPage) @@ -699,7 +701,7 @@ protected static PageReader getPageReaderMock(List dataPages, @Nullabl return ((DataPageV2) page).getDataEncoding(); }) .allMatch(encoding -> encoding == PLAIN_DICTIONARY || encoding == RLE_DICTIONARY), - hasNoNulls); + hasNoNulls, Optional.empty(), -1, -1); } private DataPage createDataPage(DataPageVersion version, ParquetEncoding encoding, ValuesWriter writer, int valueCount) @@ -713,7 +715,7 @@ private DataPage createDataPage(DataPageVersion version, ParquetEncoding encodin { Slice slice = Slices.wrappedBuffer(writer.getBytes().toByteArray()); if (version == V1) { - return new DataPageV1(slice, valueCount, slice.length(), firstRowIndex, RLE, BIT_PACKED, encoding); + return new DataPageV1(slice, valueCount, slice.length(), firstRowIndex, RLE, BIT_PACKED, encoding, 0); } return new DataPageV2( valueCount, @@ -726,7 +728,8 @@ private DataPage createDataPage(DataPageVersion version, ParquetEncoding encodin slice.length(), firstRowIndex, null, - false); + false, + 0); } private static ValuesWriter getLevelsWriter(int maxLevel, int valueCount) diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestByteStreamSplitEncoding.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestByteStreamSplitEncoding.java index fc0b69b4dd60..3fbe637274e9 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestByteStreamSplitEncoding.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestByteStreamSplitEncoding.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.util.List; +import java.util.Optional; import java.util.stream.IntStream; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; @@ -48,7 +49,7 @@ public void testReadFloatDouble() ParquetDataSource dataSource = new FileParquetDataSource( new File(Resources.getResource("byte_stream_split_float_and_double.parquet").toURI()), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames); readAndCompare(reader, getExpectedValues()); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestInt96Timestamp.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestInt96Timestamp.java index e56ac5c16098..ff8bea8c74e5 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestInt96Timestamp.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestInt96Timestamp.java @@ -112,7 +112,7 @@ public void testNanosOutsideDayRange() ParquetDataSource dataSource = new FileParquetDataSource( new File(Resources.getResource("int96_timestamps_nanos_outside_day_range.parquet").toURI()), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames); SourcePage page = reader.nextPage(); @@ -166,11 +166,12 @@ private void testVariousTimestamps(TimestampType type) slice.length(), OptionalLong.empty(), null, - false); + false, + 0); // Read and assert ColumnReaderFactory columnReaderFactory = new ColumnReaderFactory(DateTimeZone.UTC, ParquetReaderOptions.defaultOptions()); ColumnReader reader = columnReaderFactory.create(field, newSimpleAggregatedMemoryContext()); - PageReader pageReader = new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, List.of(dataPage).iterator(), false, false); + PageReader pageReader = new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, List.of(dataPage).iterator(), false, false, Optional.empty(), -1, -1); reader.setPageReader(pageReader, Optional.empty()); reader.prepareNextRead(valueCount); Block block = reader.readPrimitive().getBlock(); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java index 102e2b4fc01b..e75be3185a20 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestPageReader.java @@ -384,7 +384,6 @@ private static byte[] compress(CompressionCodec compressionCodec, byte[] bytes, } private static PageReader createPageReader(int valueCount, CompressionCodec compressionCodec, boolean hasDictionary, List slices) - throws IOException { EncodingStats.Builder encodingStats = new EncodingStats.Builder(); if (hasDictionary) { @@ -409,6 +408,7 @@ private static PageReader createPageReader(int valueCount, CompressionCodec comp columnChunkMetaData, new ColumnDescriptor(new String[] {}, new PrimitiveType(REQUIRED, INT32, ""), 0, 0), null, + Optional.empty(), Optional.empty()); } diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java index 2cd1056755e1..4e2db72c471c 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestParquetReader.java @@ -44,6 +44,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.parquet.ParquetTestUtils.createParquetReader; @@ -76,7 +77,7 @@ public void testColumnReaderMemoryUsage() columnNames, generateInputPages(types, 100, 5)), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); assertThat(parquetMetadata.getBlocks().size()).isGreaterThan(1); // Verify file has only non-dictionary encodings as dictionary memory usage is already tested in TestFlatColumnReader#testMemoryUsage parquetMetadata.getBlocks().forEach(block -> { @@ -128,7 +129,7 @@ public void testEmptyRowRangesWithColumnIndex() ParquetDataSource dataSource = new FileParquetDataSource( new File(Resources.getResource("lineitem_sorted_by_shipdate/data.parquet").toURI()), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); assertThat(parquetMetadata.getBlocks()).hasSize(2); // The predicate and the file are prepared so that page indexes will result in non-overlapping row ranges and eliminate the entire first row group // while the second row group still has to be read @@ -201,20 +202,20 @@ void testReadMetadataWithSplitOffset() ParquetReaderOptions.defaultOptions()); // Read both columns, 1 row group - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); List columnBlocks = parquetMetadata.getBlocks(0, 800); assertThat(columnBlocks.size()).isEqualTo(1); assertThat(columnBlocks.getFirst().columns().size()).isEqualTo(2); assertThat(columnBlocks.getFirst().rowCount()).isEqualTo(100); // Read both columns, half row groups - parquetMetadata = MetadataReader.readFooter(dataSource); + parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); columnBlocks = parquetMetadata.getBlocks(0, 2500); assertThat(columnBlocks.stream().allMatch(block -> block.columns().size() == 2)).isTrue(); assertThat(columnBlocks.stream().mapToLong(BlockMetadata::rowCount).sum()).isEqualTo(300); // Read both columns, all row groups - parquetMetadata = MetadataReader.readFooter(dataSource); + parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); columnBlocks = parquetMetadata.getBlocks(); assertThat(columnBlocks.stream().allMatch(block -> block.columns().size() == 2)).isTrue(); assertThat(columnBlocks.stream().mapToLong(BlockMetadata::rowCount).sum()).isEqualTo(500); @@ -238,7 +239,7 @@ void testMaxFooterReadSize() generateInputPages(types, 10, 50)), ParquetReaderOptions.defaultOptions()); - assertThatThrownBy(() -> MetadataReader.readFooter(dataSource, DataSize.ofBytes(1000))) + assertThatThrownBy(() -> MetadataReader.readFooter(dataSource, DataSize.ofBytes(1000), Optional.empty())) .hasMessageMatching(".* Parquet footer size .* exceeds maximum allowed size .*"); } @@ -248,7 +249,7 @@ private void testReadingOldParquetFiles(File file, List columnNames, Typ ParquetDataSource dataSource = new FileParquetDataSource( file, ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), ImmutableList.of(columnType), columnNames)) { SourcePage page = reader.nextPage(); Iterator expected = expectedValues.iterator(); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestTimeMillis.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestTimeMillis.java index ecc23ff12fa1..840d07b4eb28 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestTimeMillis.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/TestTimeMillis.java @@ -27,6 +27,7 @@ import java.io.File; import java.util.List; +import java.util.Optional; import static io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static io.trino.parquet.ParquetTestUtils.createParquetReader; @@ -58,7 +59,7 @@ private void testTimeMillsInt32(TimeType timeType) ParquetDataSource dataSource = new FileParquetDataSource( new File(Resources.getResource("time_millis_int32.snappy.parquet").toURI()), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); ParquetReader reader = createParquetReader(dataSource, parquetMetadata, newSimpleAggregatedMemoryContext(), types, columnNames); SourcePage page = reader.nextPage(); diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java index d6b90ff16e20..ec99cf2a4a43 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/reader/flat/TestFlatColumnReader.java @@ -137,8 +137,9 @@ private static PageReader getSimplePageReaderMock(ParquetEncoding encoding) OptionalLong.empty(), encoding, encoding, - PLAIN)); - return new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, pages.iterator(), false, false); + PLAIN, + 0)); + return new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, pages.iterator(), false, false, Optional.empty(), -1, -1); } private static PageReader getNullOnlyPageReaderMock() @@ -154,7 +155,8 @@ private static PageReader getNullOnlyPageReaderMock() OptionalLong.empty(), RLE, RLE, - PLAIN)); - return new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, pages.iterator(), false, false); + PLAIN, + 0)); + return new PageReader(new ParquetDataSourceId("test"), UNCOMPRESSED, pages.iterator(), false, false, Optional.empty(), -1, -1); } } diff --git a/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java b/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java index d004ec939a11..b35d81ee2a7f 100644 --- a/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java +++ b/lib/trino-parquet/src/test/java/io/trino/parquet/writer/TestParquetWriter.java @@ -77,6 +77,7 @@ import static io.trino.parquet.ParquetTestUtils.createParquetWriter; import static io.trino.parquet.ParquetTestUtils.generateInputPages; import static io.trino.parquet.ParquetTestUtils.writeParquetFile; +import static io.trino.parquet.metadata.HiddenColumnChunkMetadata.isHiddenColumn; import static io.trino.parquet.writer.ParquetWriterOptions.DEFAULT_BLOOM_FILTER_FPP; import static io.trino.parquet.writer.ParquetWriters.BLOOM_FILTER_EXPECTED_ENTRIES; import static io.trino.spi.type.BigintType.BIGINT; @@ -130,7 +131,7 @@ public void testWrittenPageSize() columnNames, generateInputPages(types, 100, 1000)), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); assertThat(parquetMetadata.getBlocks()).hasSize(1); assertThat(parquetMetadata.getBlocks().get(0).rowCount()).isEqualTo(100 * 1000); @@ -144,6 +145,7 @@ public void testWrittenPageSize() chunkMetaData, new ColumnDescriptor(new String[] {"columna"}, new PrimitiveType(REQUIRED, INT32, "columna"), 0, 0), null, + Optional.empty(), Optional.empty()); pageReader.readDictionaryPage(); @@ -179,7 +181,7 @@ public void testWrittenPageValueCount() columnNames, generateInputPages(types, 100, 1000)), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); assertThat(parquetMetadata.getBlocks()).hasSize(1); assertThat(parquetMetadata.getBlocks().get(0).rowCount()).isEqualTo(100 * 1000); @@ -197,6 +199,7 @@ public void testWrittenPageValueCount() columnAMetaData, new ColumnDescriptor(new String[] {"columna"}, new PrimitiveType(REQUIRED, INT32, "columna"), 0, 0), null, + Optional.empty(), Optional.empty()); pageReader.readDictionaryPage(); @@ -216,6 +219,7 @@ public void testWrittenPageValueCount() columnAMetaData, new ColumnDescriptor(new String[] {"columnb"}, new PrimitiveType(REQUIRED, INT64, "columnb"), 0, 0), null, + Optional.empty(), Optional.empty()); pageReader.readDictionaryPage(); @@ -260,7 +264,7 @@ public void testLargeStringTruncation() ImmutableList.of(new Page(2, blockA, blockB))), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); BlockMetadata blockMetaData = getOnlyElement(parquetMetadata.getBlocks()); ColumnChunkMetadata chunkMetaData = blockMetaData.columns().get(0); @@ -293,7 +297,7 @@ public void testColumnReordering() generateInputPages(types, 100, 100)), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); assertThat(parquetMetadata.getBlocks().size()).isGreaterThanOrEqualTo(10); for (BlockMetadata blockMetaData : parquetMetadata.getBlocks()) { // Verify that the columns are stored in the same order as the metadata @@ -350,7 +354,7 @@ public void testDictionaryPageOffset() generateInputPages(types, 100, 100)), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); assertThat(parquetMetadata.getBlocks().size()).isGreaterThanOrEqualTo(1); for (BlockMetadata blockMetaData : parquetMetadata.getBlocks()) { ColumnChunkMetadata chunkMetaData = getOnlyElement(blockMetaData.columns()); @@ -399,7 +403,7 @@ void testRowGroupOffset() generateInputPages(types, 100, 10)), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); List blocks = parquetMetadata.getBlocks(); assertThat(blocks.size()).isGreaterThan(1); @@ -410,7 +414,12 @@ void testRowGroupOffset() RowGroup rowGroup = rowGroups.get(rowGroupIndex); assertThat(rowGroup.isSetFile_offset()).isTrue(); BlockMetadata blockMetadata = blocks.get(rowGroupIndex); - assertThat(blockMetadata.getStartingPos()).isEqualTo(rowGroup.getFile_offset()); + assertThat(blockMetadata.columns().stream() + .filter(column -> !isHiddenColumn(column)) + .findFirst() + .map(ColumnChunkMetadata::getStartingPos) + .orElseThrow()) + .isEqualTo(rowGroup.getFile_offset()); assertThat(blockMetadata.fileRowCountOffset()).isEqualTo(fileRowCountOffset); fileRowCountOffset += rowGroup.getNum_rows(); } @@ -434,7 +443,7 @@ public void testWriteBloomFilters(Type type, List data) generateInputPages(types, 100, data)), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); // Check that bloom filters are right after each other int bloomFilterSize = Integer.highestOneBit(BlockSplitBloomFilter.optimalNumOfBits(BLOOM_FILTER_EXPECTED_ENTRIES, DEFAULT_BLOOM_FILTER_FPP) / 8) << 1; for (BlockMetadata block : parquetMetadata.getBlocks()) { @@ -499,7 +508,7 @@ void testBloomFilterWithDictionaryFallback() .build()), ParquetReaderOptions.defaultOptions()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); BlockMetadata blockMetaData = getOnlyElement(parquetMetadata.getBlocks()); ColumnChunkMetadata chunkMetaData = getOnlyElement(blockMetaData.columns()); assertThat(chunkMetaData.getEncodingStats().hasDictionaryPages()).isTrue(); diff --git a/lib/trino-plugin-toolkit/pom.xml b/lib/trino-plugin-toolkit/pom.xml index c77de764b04e..4cb80094f8a3 100644 --- a/lib/trino-plugin-toolkit/pom.xml +++ b/lib/trino-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/ClosingBinder.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/ClosingBinder.java index bd2b40dea484..f5effbd01808 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/ClosingBinder.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/ClosingBinder.java @@ -85,6 +85,11 @@ public void registerResource(Key key, Consumer close) closeables.addBinding().toProvider(new ResourceCloser(key, close)); } + public void registerCloseable(AutoCloseable instance) + { + closeables.addBinding().toInstance(instance); + } + private static class ResourceCloser implements Provider { diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/ConnectorContextModule.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/ConnectorContextModule.java new file mode 100644 index 000000000000..ee61a5d51e40 --- /dev/null +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/ConnectorContextModule.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.base; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Tracer; +import io.trino.spi.Node; +import io.trino.spi.NodeManager; +import io.trino.spi.NodeVersion; +import io.trino.spi.PageIndexerFactory; +import io.trino.spi.PageSorter; +import io.trino.spi.VersionEmbedder; +import io.trino.spi.catalog.CatalogName; +import io.trino.spi.connector.ConnectorContext; +import io.trino.spi.connector.MetadataProvider; +import io.trino.spi.type.TypeManager; + +import static java.util.Objects.requireNonNull; + +public class ConnectorContextModule + implements Module +{ + private final String catalogName; + private final ConnectorContext context; + + public ConnectorContextModule(String catalogName, ConnectorContext context) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); + this.context = requireNonNull(context, "context is null"); + } + + @Override + public void configure(Binder binder) + { + binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); + + binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); + binder.bind(Tracer.class).toInstance(context.getTracer()); + binder.bind(Node.class).toInstance(context.getCurrentNode()); + binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getCurrentNode().getVersion())); + binder.bind(NodeManager.class).toInstance(context.getNodeManager()); + binder.bind(VersionEmbedder.class).toInstance(context.getVersionEmbedder()); + binder.bind(TypeManager.class).toInstance(context.getTypeManager()); + binder.bind(MetadataProvider.class).toInstance(context.getMetadataProvider()); + binder.bind(PageSorter.class).toInstance(context.getPageSorter()); + binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); + } +} diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/TypeDeserializerModule.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/TypeDeserializerModule.java index ab030d667c09..93e67b036a6d 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/TypeDeserializerModule.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/TypeDeserializerModule.java @@ -16,25 +16,15 @@ import com.google.inject.Binder; import com.google.inject.Module; import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; import static io.airlift.json.JsonBinder.jsonBinder; -import static java.util.Objects.requireNonNull; public class TypeDeserializerModule implements Module { - private final TypeManager typeManager; - - public TypeDeserializerModule(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - @Override public void configure(Binder binder) { - binder.bind(TypeManager.class).toInstance(typeManager); jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class); } } diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/aggregation/AggregateFunctionPatterns.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/aggregation/AggregateFunctionPatterns.java index c0380178be65..094c2608774a 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/aggregation/AggregateFunctionPatterns.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/aggregation/AggregateFunctionPatterns.java @@ -14,9 +14,9 @@ package io.trino.plugin.base.aggregation; import io.trino.matching.Captures; +import io.trino.matching.CustomPattern; import io.trino.matching.Match; import io.trino.matching.Pattern; -import io.trino.matching.PatternVisitor; import io.trino.matching.Property; import io.trino.spi.connector.AggregateFunction; import io.trino.spi.expression.ConnectorExpression; @@ -82,8 +82,7 @@ public static Pattern basicAggregation() public static Pattern> variables() { - return new Pattern<>(Optional.empty()) - { + return new CustomPattern<>(Optional.empty()) { @Override public Stream accept(Object object, Captures captures, C context) { @@ -99,7 +98,10 @@ public Stream accept(Object object, Captures captures, C context) } @Override - public void accept(PatternVisitor patternVisitor) {} + public String print() + { + return "variables()"; + } }; } } diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java index 339a4b8738a0..fc93e0d2c8cd 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/classloader/ClassLoaderSafeConnectorMetadata.java @@ -79,6 +79,7 @@ import io.trino.spi.function.LanguageFunction; import io.trino.spi.function.SchemaFunctionName; import io.trino.spi.function.table.ConnectorTableFunctionHandle; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.GrantInfo; import io.trino.spi.security.Privilege; @@ -154,6 +155,14 @@ public Optional getInsertLayout(ConnectorSession session, } } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean tableReplace) + { + try (ThreadContextClassLoader _ = new ThreadContextClassLoader(classLoader)) { + return delegate.getStatisticsCollectionMetadataForWrite(session, tableMetadata, tableReplace); + } + } + @Override public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) { @@ -211,10 +220,10 @@ public Optional getTableHandleForExecute(ConnectorS } @Override - public void executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) + public Map executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) { try (ThreadContextClassLoader _ = new ThreadContextClassLoader(classLoader)) { - delegate.executeTableExecute(session, tableExecuteHandle); + return delegate.executeTableExecute(session, tableExecuteHandle); } } @@ -282,6 +291,14 @@ public Optional getInfo(ConnectorSession session, ConnectorTableHandle t } } + @Override + public Metrics getMetrics(ConnectorSession session) + { + try (ThreadContextClassLoader _ = new ThreadContextClassLoader(classLoader)) { + return delegate.getMetrics(session); + } + } + @Override public List listTables(ConnectorSession session, Optional schemaName) { diff --git a/core/trino-main/src/main/java/io/trino/execution/DistributionSnapshot.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/metrics/DistributionSnapshot.java similarity index 65% rename from core/trino-main/src/main/java/io/trino/execution/DistributionSnapshot.java rename to lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/metrics/DistributionSnapshot.java index ca8f718b14af..5e79918fe58e 100644 --- a/core/trino-main/src/main/java/io/trino/execution/DistributionSnapshot.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/metrics/DistributionSnapshot.java @@ -11,20 +11,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.execution; +package io.trino.plugin.base.metrics; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.base.MoreObjects.ToStringHelper; import io.trino.spi.metrics.Distribution; import io.trino.spi.metrics.Metric; -import io.trino.spi.metrics.Metrics; import java.util.Locale; -import java.util.Map; import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.collect.ImmutableMap.toImmutableMap; import static java.lang.String.format; @JsonTypeInfo(use = JsonTypeInfo.Id.NONE) // Do not add @class property @@ -32,35 +29,22 @@ public record DistributionSnapshot(long total, double min, double max, double p01, double p05, double p10, double p25, double p50, double p75, double p90, double p95, double p99) implements Metric { - public static Metrics pruneMetrics(Metrics metrics) + public static DistributionSnapshot fromDistribution(Distribution distribution) { - return new Metrics(metrics.getMetrics().entrySet().stream() - .collect(toImmutableMap( - Map.Entry::getKey, - entry -> { - Metric metric = entry.getValue(); - if (metric instanceof Distribution) { - return new DistributionSnapshot((Distribution) metric); - } - return metric; - }))); - } - - public DistributionSnapshot(Distribution distribution) - { - this( + double[] percentiles = distribution.getPercentiles(1, 5, 10, 25, 50, 75, 90, 95, 99); + return new DistributionSnapshot( distribution.getTotal(), distribution.getMin(), distribution.getMax(), - distribution.getPercentile(1), - distribution.getPercentile(5), - distribution.getPercentile(10), - distribution.getPercentile(25), - distribution.getPercentile(50), - distribution.getPercentile(75), - distribution.getPercentile(90), - distribution.getPercentile(95), - distribution.getPercentile(99)); + percentiles[0], + percentiles[1], + percentiles[2], + percentiles[3], + percentiles[4], + percentiles[5], + percentiles[6], + percentiles[7], + percentiles[8]); } @Override diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/metrics/TDigestHistogram.java b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/metrics/TDigestHistogram.java index 0c668f1e0583..6413f0a797dd 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/metrics/TDigestHistogram.java +++ b/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/metrics/TDigestHistogram.java @@ -33,6 +33,10 @@ public class TDigestHistogram implements Distribution { + // This is important so that we can instruct Jackson to ignore this property + // in certain places (e.g. UiQueryResource) + public static final String DIGEST_PROPERTY = "digest"; + private final TDigest digest; public static TDigestHistogram fromValue(double value) @@ -57,7 +61,7 @@ public synchronized TDigest getDigest() return TDigest.copyOf(digest); } - @JsonProperty("digest") + @JsonProperty(DIGEST_PROPERTY) public synchronized byte[] serialize() { return digest.serialize().getBytes(); @@ -65,7 +69,7 @@ public synchronized byte[] serialize() @JsonCreator @DoNotCall - public static TDigestHistogram deserialize(@JsonProperty("digest") byte[] digest) + public static TDigestHistogram deserialize(@JsonProperty(DIGEST_PROPERTY) byte[] digest) { return new TDigestHistogram(TDigest.deserialize(wrappedBuffer(digest))); } @@ -175,9 +179,13 @@ public synchronized double getP99() } @Override - public synchronized double getPercentile(double percentile) + public synchronized double[] getPercentiles(double... percentiles) { - return digest.valueAt(percentile / 100.0); + double[] digestPercentiles = new double[percentiles.length]; + for (int i = 0; i < percentiles.length; i++) { + digestPercentiles[i] = percentiles[i] / 100.0; + } + return digest.valuesAt(digestPercentiles); } @Override diff --git a/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/metrics/TestMetrics.java b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/metrics/TestMetrics.java index 43f8f7be4464..cdbc905552c7 100644 --- a/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/metrics/TestMetrics.java +++ b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/metrics/TestMetrics.java @@ -57,8 +57,9 @@ public void testMergeHistogram() TDigestHistogram merged = (TDigestHistogram) merge(m1, m2).getMetrics().get("a"); assertThat(merged.getTotal()).isEqualTo(3L); - assertThat(merged.getPercentile(0)).isEqualTo(5.0); - assertThat(merged.getPercentile(100)).isEqualTo(10.0); + double[] mergedPercentiles = merged.getPercentiles(0, 100); + assertThat(mergedPercentiles[0]).isEqualTo(5.0); + assertThat(mergedPercentiles[1]).isEqualTo(10.0); assertThat(merged.toString()) .matches("\\{count=3, p01=5\\.00, p05=5\\.00, p10=5\\.00, p25=5\\.00, p50=7\\.50, p75=10\\.00, p90=10\\.00, p95=10\\.00, p99=10\\.00, min=5\\.00, max=10\\.00\\}"); } diff --git a/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/security/BaseFileBasedConnectorAccessControlTest.java b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/security/BaseFileBasedConnectorAccessControlTest.java index 3cd43c58d64e..44f8b69042cb 100644 --- a/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/security/BaseFileBasedConnectorAccessControlTest.java +++ b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/security/BaseFileBasedConnectorAccessControlTest.java @@ -18,8 +18,8 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; -import io.trino.plugin.base.CatalogNameModule; import io.trino.spi.QueryId; +import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.ConnectorAccessControl; import io.trino.spi.connector.ConnectorSecurityContext; import io.trino.spi.connector.ConnectorTransactionHandle; @@ -855,7 +855,7 @@ public void testRefreshing(@TempDir Path tempDir) protected ConnectorAccessControl createAccessControl(Map configProperties) { - Bootstrap bootstrap = new Bootstrap(new CatalogNameModule("test_catalog"), new FileBasedAccessControlModule()); + Bootstrap bootstrap = new Bootstrap(binder -> binder.bind(CatalogName.class).toInstance(new CatalogName("test_catalog")), new FileBasedAccessControlModule()); Injector injector = bootstrap .doNotInitializeLogging() diff --git a/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/util/TestingHttpServer.java b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/util/TestingHttpServer.java index 5d52ee44e8fe..a26537f19df5 100644 --- a/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/util/TestingHttpServer.java +++ b/lib/trino-plugin-toolkit/src/test/java/io/trino/plugin/base/util/TestingHttpServer.java @@ -48,6 +48,7 @@ public TestingHttpServer() Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .quiet() .initialize(); diff --git a/lib/trino-record-decoder/pom.xml b/lib/trino-record-decoder/pom.xml index 4bfa901c2319..a39b4155d0bb 100644 --- a/lib/trino-record-decoder/pom.xml +++ b/lib/trino-record-decoder/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/lib/trino-record-decoder/src/test/java/io/trino/decoder/json/JsonFieldDecoderTester.java b/lib/trino-record-decoder/src/test/java/io/trino/decoder/json/JsonFieldDecoderTester.java index 25af7cd430e2..6e75abcdcdc0 100644 --- a/lib/trino-record-decoder/src/test/java/io/trino/decoder/json/JsonFieldDecoderTester.java +++ b/lib/trino-record-decoder/src/test/java/io/trino/decoder/json/JsonFieldDecoderTester.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Optional; +import static com.google.common.base.Preconditions.checkArgument; import static io.trino.decoder.util.DecoderTestUtil.TESTING_SESSION; import static java.lang.String.format; import static java.nio.charset.StandardCharsets.UTF_8; @@ -34,7 +35,6 @@ import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static org.assertj.core.util.Preconditions.checkArgument; public class JsonFieldDecoderTester { diff --git a/plugin/trino-ai-functions/pom.xml b/plugin/trino-ai-functions/pom.xml index cc55fd606c79..810487388a50 100644 --- a/plugin/trino-ai-functions/pom.xml +++ b/plugin/trino-ai-functions/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-ai-functions/src/main/java/io/trino/plugin/ai/functions/AiConnectorFactory.java b/plugin/trino-ai-functions/src/main/java/io/trino/plugin/ai/functions/AiConnectorFactory.java index 83d5842eb001..3a407f120198 100644 --- a/plugin/trino-ai-functions/src/main/java/io/trino/plugin/ai/functions/AiConnectorFactory.java +++ b/plugin/trino-ai-functions/src/main/java/io/trino/plugin/ai/functions/AiConnectorFactory.java @@ -15,8 +15,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; -import io.opentelemetry.api.trace.Tracer; -import io.trino.spi.catalog.CatalogName; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; @@ -42,13 +41,11 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, new AiModule(), - binder -> { - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }); + new ConnectorContextModule(catalogName, context)); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-base-jdbc/pom.xml b/plugin/trino-base-jdbc/pom.xml index ee8ff6ee3985..05976ccb86fe 100644 --- a/plugin/trino-base-jdbc/pom.xml +++ b/plugin/trino-base-jdbc/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcConnectorFactory.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcConnectorFactory.java index c1ecfdba8697..af4d9df0a78d 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcConnectorFactory.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcConnectorFactory.java @@ -16,14 +16,10 @@ import com.google.inject.Injector; import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; -import io.opentelemetry.api.OpenTelemetry; -import io.trino.spi.Node; -import io.trino.spi.VersionEmbedder; -import io.trino.spi.catalog.CatalogName; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; import java.util.Map; import java.util.function.Supplier; @@ -60,16 +56,13 @@ public Connector create(String catalogName, Map requiredConfig, Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, - binder -> binder.bind(TypeManager.class).toInstance(context.getTypeManager()), - binder -> binder.bind(Node.class).toInstance(context.getCurrentNode()), - binder -> binder.bind(VersionEmbedder.class).toInstance(context.getVersionEmbedder()), - binder -> binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()), - binder -> binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)), + new ConnectorContextModule(catalogName, context), new JdbcModule(), module.get()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(requiredConfig) .initialize(); diff --git a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcModule.java b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcModule.java index 140caed15e96..eff08f71d29b 100644 --- a/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcModule.java +++ b/plugin/trino-base-jdbc/src/main/java/io/trino/plugin/jdbc/JdbcModule.java @@ -14,6 +14,7 @@ package io.trino.plugin.jdbc; import com.google.inject.Binder; +import com.google.inject.Inject; import com.google.inject.Key; import com.google.inject.Provider; import com.google.inject.Scopes; @@ -43,6 +44,7 @@ import static io.airlift.configuration.ConfigBinder.configBinder; import static io.trino.plugin.base.ClosingBinder.closingBinder; import static java.lang.String.format; +import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newCachedThreadPool; import static org.weakref.jmx.guice.ExportBinder.newExporter; @@ -111,7 +113,7 @@ public void setup(Binder binder) newOptionalBinder(binder, Key.get(ExecutorService.class, ForJdbcClient.class)) .setDefault() - .toInstance(newCachedThreadPool(daemonThreadsNamed(format("%s-jdbc-client-%%d", catalogName)))); + .toProvider(JdbcClientExecutorServiceProvider.class); closingBinder(binder) .registerExecutor(Key.get(ExecutorService.class, ForJdbcClient.class)); @@ -146,4 +148,22 @@ public static void bindTablePropertiesProvider(Binder binder, Class + { + private final CatalogName catalogName; + + @Inject + public JdbcClientExecutorServiceProvider(CatalogName catalogName) + { + this.catalogName = requireNonNull(catalogName, "catalogName is null"); + } + + @Override + public ExecutorService get() + { + return newCachedThreadPool(daemonThreadsNamed(format("%s-jdbc-client-%%d", catalogName))); + } + } } diff --git a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java index 185d2700279c..2b74bb300432 100644 --- a/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java +++ b/plugin/trino-base-jdbc/src/test/java/io/trino/plugin/jdbc/BaseJdbcConnectorTest.java @@ -937,72 +937,14 @@ protected TestTable createTableWithDoubleAndRealColumns(String name, List 3) " + - "ORDER BY sum DESC LIMIT 10")) - .ordered() - .isFullyPushedDown(); - } + // covered by testTopNPushdown + skipTestUnless(hasBehavior(SUPPORTS_TOPN_PUSHDOWN)); // TopN over LEFT join (enforces SINGLE TopN cannot be pushed below OUTER side of join) // We expect PARTIAL TopN on the LEFT side of join to be pushed down. @@ -1439,19 +1322,6 @@ public void testExplainAnalyzePhysicalReadWallTime() "Physical input time: .*s"); } - protected QueryAssert assertConditionallyPushedDown( - Session session, - @Language("SQL") String query, - boolean condition, - PlanMatchPattern otherwiseExpected) - { - QueryAssert queryAssert = assertThat(query(session, query)); - if (condition) { - return queryAssert.isFullyPushedDown(); - } - return queryAssert.isNotFullyPushedDown(otherwiseExpected); - } - protected QueryAssert assertJoinConditionallyPushedDown( Session session, @Language("SQL") String query, @@ -1912,8 +1782,7 @@ public void testConstantUpdateWithVarcharGreaterAndLowerPredicate() public void testDeleteWithBigintEqualityPredicate() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = newTrinoTable("test_delete_bigint", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_delete_with_bigint_equality_predicate", "AS SELECT * FROM region")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE regionkey = 1", 1); assertQuery( "SELECT regionkey, name FROM " + table.getName(), @@ -1929,8 +1798,7 @@ public void testDeleteWithBigintEqualityPredicate() public void testDeleteWithVarcharEqualityPredicate() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_varchar", "(col varchar(1), pk INT)", ImmutableList.of("'a', 1", "'A', 2", "null, 3"), "pk")) { + try (TestTable table = createTestTableForWrites("test_delete_with_varchar_equality_predicate", "(col varchar(1), pk INT)", ImmutableList.of("'a', 1", "'A', 2", "null, 3"), "pk")) { if (!hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_EQUALITY) && !hasBehavior(SUPPORTS_ROW_LEVEL_UPDATE)) { assertQueryFails("DELETE FROM " + table.getName() + " WHERE col = 'A'", MODIFYING_ROWS_MESSAGE); return; @@ -1945,8 +1813,7 @@ public void testDeleteWithVarcharEqualityPredicate() public void testDeleteWithVarcharInequalityPredicate() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_varchar", "(col varchar(1), pk int)", ImmutableList.of("'a', 0", "'A', 1", "null, 2"), "pk")) { + try (TestTable table = createTestTableForWrites("test_delete_with_varchar_inequality_predicate", "(col varchar(1), pk int)", ImmutableList.of("'a', 0", "'A', 1", "null, 2"), "pk")) { if (!hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY) && !hasBehavior(SUPPORTS_MERGE)) { assertQueryFails("DELETE FROM " + table.getName() + " WHERE col != 'A'", MODIFYING_ROWS_MESSAGE); return; @@ -1961,8 +1828,7 @@ public void testDeleteWithVarcharInequalityPredicate() public void testDeleteWithVarcharGreaterAndLowerPredicate() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_varchar", "(col varchar(1), pk int)", ImmutableList.of("'0', 0", "'a', 1", "'A', 2", "'b', 3", "null, 4"), "pk")) { + try (TestTable table = createTestTableForWrites("test_delete_with_varchar_greater_and_lower_predicate", "(col varchar(1), pk int)", ImmutableList.of("'0', 0", "'a', 1", "'A', 2", "'b', 3", "null, 4"), "pk")) { if (!hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY) && !hasBehavior(SUPPORTS_MERGE)) { assertQueryFails("DELETE FROM " + table.getName() + " WHERE col < 'A'", MODIFYING_ROWS_MESSAGE); assertQueryFails("DELETE FROM " + table.getName() + " WHERE col > 'A'", MODIFYING_ROWS_MESSAGE); diff --git a/plugin/trino-bigquery/pom.xml b/plugin/trino-bigquery/pom.xml index 7f554860e4ba..3f64001bd789 100644 --- a/plugin/trino-bigquery/pom.xml +++ b/plugin/trino-bigquery/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorFactory.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorFactory.java index 19d217723e15..f8a30f451591 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorFactory.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryConnectorFactory.java @@ -16,17 +16,12 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; import org.weakref.jmx.guice.MBeanModule; import java.util.Map; @@ -57,17 +52,11 @@ public Connector create(String catalogName, Map config, Connecto new MBeanServerModule(), new MBeanModule(), new ConnectorObjectNameGeneratorModule("io.trino.plugin.bigquery", "trino.plugin.bigquery"), - binder -> { - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }); + new ConnectorContextModule(catalogName, context)); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryMetadata.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryMetadata.java index f83d2433a5b1..c1db6b9d1f91 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryMetadata.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryMetadata.java @@ -377,8 +377,8 @@ private static boolean isBigLakeTable(TableDefinition tableDefinition) //BigLake tables are external with connectionId that don't have objectMetadata (ObjectTable discriminator) and their uri starts with gs:// (OMNI table discriminator) List sourceUris = externalTableDefinition.getSourceUris(); return !isNullOrEmpty(externalTableDefinition.getConnectionId()) && - isNullOrEmpty(externalTableDefinition.getObjectMetadata()) && - (sourceUris != null && sourceUris.stream().allMatch(uri -> uri.startsWith("gs://"))); + isNullOrEmpty(externalTableDefinition.getObjectMetadata()) && + (sourceUris != null && sourceUris.stream().allMatch(uri -> uri.startsWith("gs://"))); } return false; } diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java index 3c54f13a7db0..9072c195725e 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/BaseBigQueryConnectorTest.java @@ -103,6 +103,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_MATERIALIZED_VIEW, SUPPORTS_CREATE_VIEW, SUPPORTS_DEFAULT_COLUMN_VALUE, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MAP_TYPE, SUPPORTS_MERGE, SUPPORTS_NEGATIVE_DATE, @@ -882,7 +883,7 @@ public void testQueryLabeling() private void assertLabelForTable(String expectedView, QueryId queryId, String traceToken) { - String expectedLabel = "q_" + queryId.toString() + "__t_" + traceToken; + String expectedLabel = "q_" + queryId.id() + "__t_" + traceToken; @Language("SQL") String checkForLabelQuery = @@ -969,7 +970,7 @@ public void testWildcardTableWithDifferentColumnDefinition() assertQuery("DESCRIBE test.\"" + wildcardTable + "\"", "VALUES ('value', 'varchar', '', '')"); assertThat(query("SELECT * FROM test.\"" + wildcardTable + "\"")) - .failure().hasMessageContaining("Cannot read field of type INT64 as STRING Field: value"); + .failure().hasMessageContaining("Cannot read field of type INT64 as STRING"); } finally { onBigQuery("DROP TABLE IF EXISTS test." + firstTable); diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMappingWithCache.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMappingWithCache.java index ca4f56dbcbeb..f9fa6c950b96 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMappingWithCache.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryCaseInsensitiveMappingWithCache.java @@ -15,7 +15,11 @@ import com.google.common.collect.ImmutableMap; import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.parallel.Execution; +import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; + +@Execution(SAME_THREAD) // run single-threaded to avoid interference with other tests final class TestBigQueryCaseInsensitiveMappingWithCache extends BaseBigQueryCaseInsensitiveMapping { diff --git a/plugin/trino-blackhole/pom.xml b/plugin/trino-blackhole/pom.xml index cb96017c74dd..2f42cf8673b5 100644 --- a/plugin/trino-blackhole/pom.xml +++ b/plugin/trino-blackhole/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-cassandra/pom.xml b/plugin/trino-cassandra/pom.xml index af3d92c53077..2b129c382515 100644 --- a/plugin/trino-cassandra/pom.xml +++ b/plugin/trino-cassandra/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -262,13 +262,13 @@ org.testcontainers - cassandra + testcontainers test org.testcontainers - testcontainers + testcontainers-cassandra test diff --git a/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraClientModule.java b/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraClientModule.java index 36e604d7b5e1..37cafbbce293 100644 --- a/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraClientModule.java +++ b/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraClientModule.java @@ -61,18 +61,9 @@ public class CassandraClientModule extends AbstractConfigurationAwareModule { - private final TypeManager typeManager; - - public CassandraClientModule(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - @Override public void setup(Binder binder) { - binder.bind(TypeManager.class).toInstance(typeManager); - verifyConnectorUnsafeAllowed(binder, "cassandra"); binder.bind(CassandraConnector.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraConnectorFactory.java b/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraConnectorFactory.java index 84c674de70da..273427eac06a 100644 --- a/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraConnectorFactory.java +++ b/plugin/trino-cassandra/src/main/java/io/trino/plugin/cassandra/CassandraConnectorFactory.java @@ -16,7 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -45,13 +45,14 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, - binder -> binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()), + new ConnectorContextModule(catalogName, context), new MBeanModule(), new JsonModule(), - new CassandraClientModule(context.getTypeManager()), + new CassandraClientModule(), new MBeanServerModule()); Injector injector = app.doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraConnectorTest.java b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraConnectorTest.java index 9b20f64933e3..3ea2793f79e0 100644 --- a/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraConnectorTest.java +++ b/plugin/trino-cassandra/src/test/java/io/trino/plugin/cassandra/TestCassandraConnectorTest.java @@ -95,6 +95,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT, SUPPORTS_CREATE_VIEW, SUPPORTS_DEFAULT_COLUMN_VALUE, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MAP_TYPE, SUPPORTS_MERGE, SUPPORTS_NOT_NULL_CONSTRAINT, diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml index d3e72668a3bb..61151c141ef8 100644 --- a/plugin/trino-clickhouse/pom.xml +++ b/plugin/trino-clickhouse/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -217,19 +217,19 @@ org.testcontainers - clickhouse + testcontainers test org.testcontainers - jdbc + testcontainers-clickhouse test org.testcontainers - testcontainers + testcontainers-jdbc test diff --git a/plugin/trino-delta-lake/pom.xml b/plugin/trino-delta-lake/pom.xml index 3fe272c2325c..8bba72636aad 100644 --- a/plugin/trino-delta-lake/pom.xml +++ b/plugin/trino-delta-lake/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -540,6 +540,20 @@ org.antlr antlr4-maven-plugin + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConnectorFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConnectorFactory.java index 93d319a2cfe7..dd3723767ae5 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConnectorFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeConnectorFactory.java @@ -20,10 +20,8 @@ import io.airlift.bootstrap.Bootstrap; import io.airlift.bootstrap.LifeCycleManager; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.manager.FileSystemModule; -import io.trino.plugin.base.CatalogNameModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorAccessControl; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSinkProvider; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider; @@ -34,11 +32,6 @@ import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule; import io.trino.plugin.hive.HiveConfig; -import io.trino.plugin.hive.NodeVersion; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.PageIndexerFactory; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorAccessControl; @@ -52,7 +45,6 @@ import io.trino.spi.function.FunctionProvider; import io.trino.spi.function.table.ConnectorTableFunction; import io.trino.spi.procedure.Procedure; -import io.trino.spi.type.TypeManager; import org.weakref.jmx.guice.MBeanModule; import java.util.Map; @@ -96,26 +88,17 @@ public static Connector createConnector( new ConnectorObjectNameGeneratorModule("io.trino.plugin.deltalake", "trino.plugin.deltalake"), new JsonModule(), new MBeanServerModule(), - new CatalogNameModule(catalogName), metastoreModule.orElse(new DeltaLakeMetastoreModule()), new DeltaLakeModule(), new DeltaLakeSecurityModule(), new DeltaLakeSynchronizerModule(), - new FileSystemModule(catalogName, context.getCurrentNode().isCoordinator(), context.getOpenTelemetry(), false), - binder -> { - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getCurrentNode().getVersion())); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }, + new FileSystemModule(catalogName, context, false), + new ConnectorContextModule(catalogName, context), module); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMergeSink.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMergeSink.java index ce5543f2bcc2..b1f63d4a45de 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMergeSink.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMergeSink.java @@ -388,7 +388,7 @@ private Slice writeMergeResult(Slice path, FileDeletion deletion) } TrinoInputFile inputFile = fileSystem.newInputFile(Location.of(path.toStringUtf8())); try (ParquetDataSource dataSource = new TrinoParquetDataSource(inputFile, parquetReaderOptions, fileFormatDataSourceStats)) { - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); long rowCount = parquetMetadata.getBlocks().stream().map(BlockMetadata::rowCount).mapToLong(Long::longValue).sum(); RoaringBitmapArray rowsRetained = new RoaringBitmapArray(); rowsRetained.addRange(0, rowCount - 1); @@ -690,6 +690,7 @@ private ConnectorPageSource createParquetPageSource(Location path) new FileFormatDataSourceStats(), ParquetReaderOptions.builder().withBloomFilter(false).build(), Optional.empty(), + Optional.empty(), domainCompactionThreshold, OptionalLong.of(fileSize)); } diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java index b6e731e3552b..240be319a10d 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeMetadataFactory.java @@ -26,10 +26,10 @@ import io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointWriterManager; import io.trino.plugin.deltalake.transactionlog.reader.TransactionLogReaderFactory; import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.security.UsingSystemSecurity; import io.trino.spi.Node; +import io.trino.spi.NodeVersion; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSinkProvider.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSinkProvider.java index 36af036c33b7..0634eb5f745e 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSinkProvider.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSinkProvider.java @@ -23,8 +23,8 @@ import io.trino.plugin.deltalake.procedure.DeltaTableOptimizeHandle; import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.parquet.ParquetReaderConfig; +import io.trino.spi.NodeVersion; import io.trino.spi.PageIndexerFactory; import io.trino.spi.connector.ConnectorInsertTableHandle; import io.trino.spi.connector.ConnectorMergeSink; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java index 9e0f75ec1162..f4bff5ccec4d 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakePageSourceProvider.java @@ -263,6 +263,7 @@ public ConnectorPageSource createPageSource( fileFormatDataSourceStats, options, Optional.empty(), + Optional.empty(), domainCompactionThreshold, OptionalLong.of(split.getFileSize())); @@ -362,7 +363,7 @@ private static PositionDeleteFilter readDeletes( private Map loadParquetIdAndNameMapping(TrinoInputFile inputFile, ParquetReaderOptions options) { try (ParquetDataSource dataSource = new TrinoParquetDataSource(inputFile, options, fileFormatDataSourceStats)) { - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize(), Optional.empty()); FileMetadata fileMetaData = parquetMetadata.getFileMetaData(); MessageType fileSchema = fileMetaData.getSchema(); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java index b192f03727e1..131f603f7c95 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/DeltaLakeWriter.java @@ -179,7 +179,7 @@ public DataFileInfo getDataFileInfo() { Location path = rootTableLocation.appendPath(relativeFilePath); FileMetaData fileMetaData = fileWriter.getFileMetadata(); - ParquetMetadata parquetMetadata = new ParquetMetadata(fileMetaData, new ParquetDataSourceId(path.toString())); + ParquetMetadata parquetMetadata = new ParquetMetadata(fileMetaData, new ParquetDataSourceId(path.toString()), Optional.empty()); return new DataFileInfo( relativeFilePath, diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesFunctionProcessor.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesFunctionProcessor.java index f52167042a77..c697c8256542 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesFunctionProcessor.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/functions/tablechanges/TableChangesFunctionProcessor.java @@ -206,6 +206,7 @@ private static ConnectorPageSource createDeltaLakePageSource( fileFormatDataSourceStats, parquetReaderOptions, Optional.empty(), + Optional.empty(), domainCompactionThreshold, OptionalLong.empty()); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java index fd2e8fcb6206..b26e0e263486 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/TableSnapshot.java @@ -350,34 +350,51 @@ private Stream getV2CheckpointTransactionLogEntrie { // Sidecar files contain only ADD and REMOVE entry types. https://github.com/delta-io/delta/blob/master/PROTOCOL.md#v2-spec Set dataEntryTypes = Sets.intersection(entryTypes, Set.of(ADD, REMOVE)); - List>> logEntryStreamFutures = - getV2CheckpointEntries(session, entryTypes, metadataEntry, protocolEntry, checkpointSchemaManager, typeManager, stats, checkpoint, checkpointFile, partitionConstraint, addStatsMinMaxColumnFilter, fileSystem, fileSize) - .map(v2checkpointEntry -> { - if (v2checkpointEntry.getSidecar() == null || dataEntryTypes.isEmpty()) { - return CompletableFuture.completedFuture(Stream.of(v2checkpointEntry)); - } - // Sidecar files are retrieved in parallel using a bounded executor - return supplyAsync(() -> { - Location sidecar = checkpointFile.location().sibling("_sidecars").appendPath(v2checkpointEntry.getSidecar().path()); - CheckpointEntryIterator iterator = new CheckpointEntryIterator( - fileSystem.newInputFile(sidecar), - session, - v2checkpointEntry.getSidecar().sizeInBytes(), - checkpointSchemaManager, - typeManager, - dataEntryTypes, - metadataEntry, - protocolEntry, - stats, - parquetReaderOptions, - checkpointRowStatisticsWritingEnabled, - domainCompactionThreshold, - partitionConstraint, - addStatsMinMaxColumnFilter); - return stream(iterator).onClose(iterator::close); - }, executor); - }) - .collect(toImmutableList()); + Stream v2CheckpointEntries = getV2CheckpointEntries( + session, + entryTypes, + metadataEntry, + protocolEntry, + checkpointSchemaManager, + typeManager, + stats, + checkpoint, + checkpointFile, + partitionConstraint, + addStatsMinMaxColumnFilter, + fileSystem, + fileSize); + if (dataEntryTypes.isEmpty()) { + return v2CheckpointEntries; + } + + List>> logEntryStreamFutures = v2CheckpointEntries + .map(v2checkpointEntry -> { + if (v2checkpointEntry.getSidecar() == null) { + return CompletableFuture.completedFuture(Stream.of(v2checkpointEntry)); + } + // Sidecar files are retrieved in parallel using a bounded executor + return supplyAsync(() -> { + Location sidecar = checkpointFile.location().sibling("_sidecars").appendPath(v2checkpointEntry.getSidecar().path()); + CheckpointEntryIterator iterator = new CheckpointEntryIterator( + fileSystem.newInputFile(sidecar), + session, + v2checkpointEntry.getSidecar().sizeInBytes(), + checkpointSchemaManager, + typeManager, + dataEntryTypes, + metadataEntry, + protocolEntry, + stats, + parquetReaderOptions, + checkpointRowStatisticsWritingEnabled, + domainCompactionThreshold, + partitionConstraint, + addStatsMinMaxColumnFilter); + return stream(iterator).onClose(iterator::close); + }, executor); + }) + .collect(toImmutableList()); // Return the stream to retrieve the values of the futures lazily and allow streamlined split generation return logEntryStreamFutures.stream() .mapMulti((logEntryStream, builder) diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointEntryIterator.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointEntryIterator.java index dd62e0eeba44..a8aaeda155ae 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointEntryIterator.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointEntryIterator.java @@ -219,6 +219,7 @@ public CheckpointEntryIterator( stats, parquetReaderOptions, Optional.empty(), + Optional.empty(), domainCompactionThreshold, OptionalLong.of(fileSize)); diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java index aa25ab7fe384..8278148323e2 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/checkpoint/CheckpointWriterManager.java @@ -29,7 +29,7 @@ import io.trino.plugin.deltalake.transactionlog.TableSnapshot; import io.trino.plugin.deltalake.transactionlog.TableSnapshot.MetadataAndProtocolEntry; import io.trino.plugin.deltalake.transactionlog.TransactionLogAccess; -import io.trino.plugin.hive.NodeVersion; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.SchemaTableName; diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/writer/FileSystemTransactionLogWriter.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/writer/FileSystemTransactionLogWriter.java index c79301d9272b..cfd9bbf56484 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/writer/FileSystemTransactionLogWriter.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/transactionlog/writer/FileSystemTransactionLogWriter.java @@ -33,11 +33,11 @@ import java.util.List; import java.util.Optional; +import static com.google.common.base.Preconditions.checkState; import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.getTransactionLogDir; import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.getTransactionLogJsonEntryPath; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; -import static org.apache.parquet.Preconditions.checkState; public class FileSystemTransactionLogWriter implements TransactionLogWriter diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java index 6756c069b330..aa283942f7da 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeBasic.java @@ -141,6 +141,7 @@ public class TestDeltaLakeBasic new ResourceTable("liquid_clustering", "deltalake/liquid_clustering"), new ResourceTable("region_91_lts", "databricks91/region"), new ResourceTable("region_104_lts", "databricks104/region"), + new ResourceTable("region_113_lts", "databricks113/region"), new ResourceTable("timestamp_ntz", "databricks131/timestamp_ntz"), new ResourceTable("timestamp_ntz_partition", "databricks131/timestamp_ntz_partition"), new ResourceTable("uniform_hudi", "deltalake/uniform_hudi"), @@ -243,6 +244,14 @@ void testDatabricks104() .matches("SELECT * FROM tpch.tiny.region"); } + @Test + void testDatabricks113() + { + assertThat(query("SELECT * FROM region_113_lts")) + .skippingTypesCheck() // name and comment columns are unbounded varchar in Delta Lake and bounded varchar in TPCH + .matches("SELECT * FROM tpch.tiny.region"); + } + @Test public void testNoColumnStats() { @@ -405,7 +414,7 @@ private void testPartitionValuesParsedCheckpoint(ColumnMappingMode columnMapping assertThat(partitionValuesParsedType.getFields().stream().collect(onlyElement()).getName().orElseThrow()).isEqualTo(physicalColumnName); TrinoParquetDataSource dataSource = new TrinoParquetDataSource(new LocalInputFile(checkpoint.toFile()), ParquetReaderOptions.defaultOptions(), new FileFormatDataSourceStats()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); try (ParquetReader reader = createParquetReader(dataSource, parquetMetadata, ImmutableList.of(addEntryType), List.of("add"))) { List actual = new ArrayList<>(); SourcePage page = reader.nextPage(); @@ -483,7 +492,8 @@ private void testOptimizeWithColumnMappingMode(String columnMappingMode) // Verify optimized parquet file contains the expected physical id and name TrinoInputFile inputFile = new LocalInputFile(tableLocation.resolve(addFileEntry.getPath()).toFile()); ParquetMetadata parquetMetadata = MetadataReader.readFooter( - new TrinoParquetDataSource(inputFile, ParquetReaderOptions.defaultOptions(), new FileFormatDataSourceStats())); + new TrinoParquetDataSource(inputFile, ParquetReaderOptions.defaultOptions(), new FileFormatDataSourceStats()), + Optional.empty()); FileMetadata fileMetaData = parquetMetadata.getFileMetaData(); PrimitiveType physicalType = getOnlyElement(fileMetaData.getSchema().getColumns().iterator()).getPrimitiveType(); assertThat(physicalType.getName()).isEqualTo(physicalName); @@ -1640,10 +1650,10 @@ public void testVariantReadNull() assertThat(query("TABLE " + tableName)) .skippingTypesCheck() .matches("VALUES " + - "(1, JSON '{\"a\":1}', MAP(ARRAY['key1'], ARRAY[NULL]))," + - "(2, JSON '{\"a\":2}', MAP(ARRAY['key1'], ARRAY[JSON '{\"key\":\"value\"}']))," + - "(3, JSON 'null', NULL)," + - "(4, NULL, NULL)," + + "(1, JSON '{\"a\":1}', MAP(ARRAY['key1'], ARRAY[NULL]))," + + "(2, JSON '{\"a\":2}', MAP(ARRAY['key1'], ARRAY[JSON '{\"key\":\"value\"}']))," + + "(3, JSON 'null', NULL)," + + "(4, NULL, NULL)," + "(5, JSON '{\"a\":5}', NULL)"); } diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java index 99236e23d545..655faa079291 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeMetadata.java @@ -22,7 +22,6 @@ import com.google.inject.Scopes; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.cache.CachingHostAddressProvider; import io.trino.filesystem.cache.DefaultCachingHostAddressProvider; @@ -32,21 +31,16 @@ import io.trino.metastore.Database; import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.RawHiveMetastoreFactory; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastore; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule; import io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore; import io.trino.plugin.deltalake.transactionlog.MetadataEntry; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; -import io.trino.plugin.hive.NodeVersion; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.PageIndexerFactory; import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.Assignment; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; -import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorTableHandle; import io.trino.spi.connector.ConnectorTableLayout; import io.trino.spi.connector.ConnectorTableMetadata; @@ -63,7 +57,6 @@ import io.trino.spi.type.DoubleType; import io.trino.spi.type.RowType; import io.trino.spi.type.Type; -import io.trino.spi.type.TypeManager; import io.trino.spi.type.VarcharType; import io.trino.testing.TestingConnectorContext; import io.trino.tests.BogusType; @@ -189,19 +182,11 @@ public void setUp() .put("hive.metastore.catalog.dir", temporaryCatalogDirectory.getPath()) .buildOrThrow(); + TestingConnectorContext context = new TestingConnectorContext(); Bootstrap app = new Bootstrap( // connector dependencies new JsonModule(), - binder -> { - ConnectorContext context = new TestingConnectorContext(); - binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getCurrentNode().getVersion())); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(CatalogName.class).toInstance(new CatalogName("test")); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - }, + new ConnectorContextModule("test", context), // connector modules new DeltaLakeSecurityModule(), new DeltaLakeMetastoreModule(), diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePageSink.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePageSink.java index f699f51bc289..fe8d16403680 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePageSink.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakePageSink.java @@ -26,8 +26,8 @@ import io.trino.plugin.deltalake.metastore.NoOpVendedCredentialsProvider; import io.trino.plugin.deltalake.transactionlog.ProtocolEntry; import io.trino.plugin.hive.HiveTransactionHandle; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.parquet.ParquetReaderConfig; +import io.trino.spi.NodeVersion; import io.trino.spi.Page; import io.trino.spi.PageBuilder; import io.trino.spi.block.BlockBuilder; diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java index 8f138b960b5d..efbcd768c055 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeSplitManager.java @@ -44,9 +44,9 @@ import io.trino.plugin.deltalake.transactionlog.writer.NoIsolationSynchronizer; import io.trino.plugin.deltalake.transactionlog.writer.TransactionLogSynchronizerManager; import io.trino.plugin.hive.HiveTransactionHandle; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hive.parquet.ParquetWriterConfig; +import io.trino.spi.NodeVersion; import io.trino.spi.SplitWeight; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorSplit; @@ -221,7 +221,7 @@ public Stream getActiveFiles( newDirectExecutorService()); TransactionLogReaderFactory transactionLogReaderFactory = new FileSystemTransactionLogReaderFactory(fileSystemFactory); - HiveMetastoreFactory hiveMetastoreFactory = HiveMetastoreFactory.ofInstance(createTestingFileHiveMetastore(new MemoryFileSystemFactory(), Location.of("memory:///"))); + HiveMetastoreFactory hiveMetastoreFactory = HiveMetastoreFactory.ofInstance(createTestingFileHiveMetastore(new MemoryFileSystemFactory(), Location.of("memory:///")), false); DeltaLakeMetadataFactory metadataFactory = new DeltaLakeMetadataFactory( hiveMetastoreFactory, new DefaultDeltaLakeFileSystemFactory(hdfsFileSystemFactory, new NoOpVendedCredentialsProvider()), diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java index 07bf6000f02d..44277e5669eb 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/TestingDeltaLakeMetastoreModule.java @@ -41,7 +41,7 @@ public TestingDeltaLakeMetastoreModule(HiveMetastore metastore) @Override public void setup(Binder binder) { - binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore)); + binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore, false)); install(new CachingHiveMetastoreModule()); binder.bind(DeltaLakeTableOperationsProvider.class).to(DeltaLakeFileMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java index c9d114bc5856..b326af431e7b 100644 --- a/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java +++ b/plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java @@ -22,8 +22,6 @@ import io.airlift.bootstrap.Bootstrap; import io.airlift.bootstrap.LifeCycleManager; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.manager.FileSystemModule; import io.trino.metastore.Column; import io.trino.metastore.Database; @@ -31,22 +29,18 @@ import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.deltalake.DeltaLakeMetadata; import io.trino.plugin.deltalake.DeltaLakeMetadataFactory; import io.trino.plugin.deltalake.DeltaLakeModule; import io.trino.plugin.deltalake.DeltaLakeSecurityModule; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule; -import io.trino.plugin.hive.NodeVersion; -import io.trino.spi.Node; -import io.trino.spi.PageIndexerFactory; import io.trino.spi.TrinoException; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.RelationColumnsMetadata; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.SchemaTablePrefix; -import io.trino.spi.type.TypeManager; import io.trino.testing.TestingConnectorContext; import io.trino.testing.TestingConnectorSession; import org.junit.jupiter.api.AfterAll; @@ -118,21 +112,13 @@ public void setUp() Bootstrap app = new Bootstrap( // connector dependencies new JsonModule(), - binder -> { - binder.bind(CatalogName.class).toInstance(new CatalogName("test")); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); - binder.bind(NodeVersion.class).toInstance(new NodeVersion("test_version")); - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - }, + new ConnectorContextModule("test", context), // connector modules new DeltaLakeMetastoreModule(), new DeltaLakeModule(), new DeltaLakeSecurityModule(), // test setup - new FileSystemModule("test", context.getCurrentNode().isCoordinator(), context.getOpenTelemetry(), false)); + new FileSystemModule("test", context, false)); Injector injector = app .doNotInitializeLogging() diff --git a/plugin/trino-delta-lake/src/test/resources/databricks113/region/README.md b/plugin/trino-delta-lake/src/test/resources/databricks113/region/README.md new file mode 100644 index 000000000000..880f97103d41 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks113/region/README.md @@ -0,0 +1,13 @@ +Data generated using Databricks 11.3: + +```sql +CREATE TABLE default.region (regionkey bigint, name string, comment string) +USING DELTA LOCATION 's3://trino-ci-test/default/region'; + +INSERT INTO default.region VALUES +(0, 'AFRICA', 'lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to '), +(1, 'AMERICA', 'hs use ironic, even requests. s'), +(2, 'ASIA', 'ges. thinly even pinto beans ca'), +(3, 'EUROPE', 'ly final courts cajole furiously final excuse'), +(4, 'MIDDLE EAST', 'uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl'); +``` diff --git a/plugin/trino-delta-lake/src/test/resources/databricks113/region/_delta_log/00000000000000000000.json b/plugin/trino-delta-lake/src/test/resources/databricks113/region/_delta_log/00000000000000000000.json new file mode 100644 index 000000000000..6b63a5a572fc --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks113/region/_delta_log/00000000000000000000.json @@ -0,0 +1,3 @@ +{"commitInfo":{"timestamp":1760486422335,"userId":"7853186923043731","userName":"yuya.ebihara@starburstdata.com","operation":"CREATE TABLE","operationParameters":{"isManaged":"false","description":null,"partitionBy":"[]","properties":"{}"},"notebook":{"notebookId":"1841155838656679"},"clusterId":"0705-101014-4m31mz15","isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"engineInfo":"Databricks-Runtime/11.3.x-scala2.12","txnId":"0aa0cbaa-2347-4272-934f-c451156e814f"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"743428f4-7b16-48e9-ad86-729b370babca","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"regionkey\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"comment\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1760486422096}} diff --git a/plugin/trino-delta-lake/src/test/resources/databricks113/region/_delta_log/00000000000000000001.json b/plugin/trino-delta-lake/src/test/resources/databricks113/region/_delta_log/00000000000000000001.json new file mode 100644 index 000000000000..fd06bb398d91 --- /dev/null +++ b/plugin/trino-delta-lake/src/test/resources/databricks113/region/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"commitInfo":{"timestamp":1760486426498,"userId":"7853186923043731","userName":"yuya.ebihara@starburstdata.com","operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"notebook":{"notebookId":"1841155838656679"},"clusterId":"0705-101014-4m31mz15","readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"5","numOutputBytes":"1657"},"engineInfo":"Databricks-Runtime/11.3.x-scala2.12","txnId":"eeda1a32-7bd5-4ec8-a232-0b97079df819"}} +{"add":{"path":"part-00000-ae6566cb-052d-42a8-9e45-33c383cb2108-c000.snappy.parquet","partitionValues":{},"size":1657,"modificationTime":1760486427000,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"regionkey\":0,\"name\":\"AFRICA\",\"comment\":\"ges. thinly even pinto beans ca\"},\"maxValues\":{\"regionkey\":4,\"name\":\"MIDDLE EAST\",\"comment\":\"uickly special accounts cajole c\"},\"nullCount\":{\"regionkey\":0,\"name\":0,\"comment\":0}}","tags":{"INSERTION_TIME":"1760486427000000","MIN_INSERTION_TIME":"1760486427000000","MAX_INSERTION_TIME":"1760486427000000","OPTIMIZE_TARGET_SIZE":"268435456"}}} diff --git a/plugin/trino-delta-lake/src/test/resources/databricks113/region/part-00000-ae6566cb-052d-42a8-9e45-33c383cb2108-c000.snappy.parquet b/plugin/trino-delta-lake/src/test/resources/databricks113/region/part-00000-ae6566cb-052d-42a8-9e45-33c383cb2108-c000.snappy.parquet new file mode 100644 index 000000000000..87b608dd7ff6 Binary files /dev/null and b/plugin/trino-delta-lake/src/test/resources/databricks113/region/part-00000-ae6566cb-052d-42a8-9e45-33c383cb2108-c000.snappy.parquet differ diff --git a/plugin/trino-druid/pom.xml b/plugin/trino-druid/pom.xml index c16ac91c19ce..0478c663d278 100644 --- a/plugin/trino-druid/pom.xml +++ b/plugin/trino-druid/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-druid/src/test/java/io/trino/plugin/druid/TestDruidConnectorTest.java b/plugin/trino-druid/src/test/java/io/trino/plugin/druid/TestDruidConnectorTest.java index 399d4f051bd3..503c7f79a4aa 100644 --- a/plugin/trino-druid/src/test/java/io/trino/plugin/druid/TestDruidConnectorTest.java +++ b/plugin/trino-druid/src/test/java/io/trino/plugin/druid/TestDruidConnectorTest.java @@ -18,12 +18,8 @@ import io.trino.plugin.jdbc.BaseJdbcConnectorTest; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.SchemaTableName; -import io.trino.sql.planner.assertions.PlanMatchPattern; import io.trino.sql.planner.plan.AggregationNode; import io.trino.sql.planner.plan.FilterNode; -import io.trino.sql.planner.plan.JoinNode; -import io.trino.sql.planner.plan.TableScanNode; -import io.trino.sql.planner.plan.TopNNode; import io.trino.testing.MaterializedResult; import io.trino.testing.QueryRunner; import io.trino.testing.TestingConnectorBehavior; @@ -39,8 +35,6 @@ import static io.trino.plugin.druid.DruidQueryRunner.copyAndIngestTpchData; import static io.trino.plugin.druid.DruidTpchTables.SELECT_FROM_ORDERS; import static io.trino.spi.type.VarcharType.VARCHAR; -import static io.trino.sql.planner.assertions.PlanMatchPattern.anyTree; -import static io.trino.sql.planner.assertions.PlanMatchPattern.node; import static io.trino.sql.planner.assertions.PlanMatchPattern.output; import static io.trino.sql.planner.assertions.PlanMatchPattern.values; import static io.trino.testing.MaterializedResult.resultBuilder; @@ -234,47 +228,6 @@ public void testFilteringForTablesAndColumns() assertThat(query("DESCRIBE " + datasourceB)).result().matches(expectedColumns); } - @Test - public void testLimitPushDown() - { - assertThat(query("SELECT name FROM nation LIMIT 30")).isFullyPushedDown(); // Use high limit for result determinism - - // with filter over numeric column - assertThat(query("SELECT name FROM nation WHERE regionkey = 3 LIMIT 5")).isFullyPushedDown(); - - // with filter over varchar column - assertThat(query("SELECT name FROM nation WHERE name < 'EEE' LIMIT 5")).isFullyPushedDown(); - - // with aggregation - assertThat(query("SELECT max(regionkey) FROM nation LIMIT 5")).isNotFullyPushedDown(AggregationNode.class); // global aggregation, LIMIT removed TODO https://github.com/trinodb/trino/pull/4313 - assertThat(query("SELECT regionkey, max(name) FROM nation GROUP BY regionkey LIMIT 5")).isNotFullyPushedDown(AggregationNode.class); // TODO https://github.com/trinodb/trino/pull/4313 - - // distinct limit can be pushed down even without aggregation pushdown - assertThat(query("SELECT DISTINCT regionkey FROM nation LIMIT 5")).isFullyPushedDown(); - - // with aggregation and filter over numeric column - assertThat(query("SELECT regionkey, count(*) FROM nation WHERE nationkey < 5 GROUP BY regionkey LIMIT 3")).isNotFullyPushedDown(AggregationNode.class); // TODO https://github.com/trinodb/trino/pull/4313 - // with aggregation and filter over varchar column - assertThat(query("SELECT regionkey, count(*) FROM nation WHERE name < 'EGYPT' GROUP BY regionkey LIMIT 3")).isNotFullyPushedDown(AggregationNode.class); // TODO https://github.com/trinodb/trino/pull/4313 - - // with TopN over numeric column - assertThat(query("SELECT * FROM (SELECT regionkey FROM nation ORDER BY nationkey ASC LIMIT 10) LIMIT 5")).isNotFullyPushedDown(TopNNode.class); - // with TopN over varchar column - assertThat(query("SELECT * FROM (SELECT regionkey FROM nation ORDER BY name ASC LIMIT 10) LIMIT 5")).isNotFullyPushedDown(TopNNode.class); - - // with join - PlanMatchPattern joinOverTableScans = node(JoinNode.class, - anyTree(node(TableScanNode.class)), - anyTree(node(TableScanNode.class))); - assertThat(query( - joinPushdownEnabled(getSession()), - "SELECT n.name, r.name " + - "FROM nation n " + - "LEFT JOIN region r USING (regionkey) " + - "LIMIT 30")) - .isNotFullyPushedDown(joinOverTableScans); - } - @Test @Override public void testInsertNegativeDate() diff --git a/plugin/trino-duckdb/pom.xml b/plugin/trino-duckdb/pom.xml index 95703154dbe9..ad986f3c8482 100644 --- a/plugin/trino-duckdb/pom.xml +++ b/plugin/trino-duckdb/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-elasticsearch/pom.xml b/plugin/trino-elasticsearch/pom.xml index 4fc2965c3a6b..2bddf99c0456 100644 --- a/plugin/trino-elasticsearch/pom.xml +++ b/plugin/trino-elasticsearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -193,6 +193,11 @@ aws-core + + software.amazon.awssdk + http-auth-aws + + software.amazon.awssdk http-client-spi @@ -398,19 +403,19 @@ org.testcontainers - elasticsearch + testcontainers test org.testcontainers - nginx + testcontainers-elasticsearch test org.testcontainers - testcontainers + testcontainers-nginx test diff --git a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ElasticsearchConnectorFactory.java b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ElasticsearchConnectorFactory.java index 22da4734f5fd..6864a9500210 100644 --- a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ElasticsearchConnectorFactory.java +++ b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/ElasticsearchConnectorFactory.java @@ -16,11 +16,10 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.spi.Node; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; @@ -55,15 +54,13 @@ public Connector create(String catalogName, Map config, Connecto new MBeanServerModule(), new ConnectorObjectNameGeneratorModule("io.trino.plugin.elasticsearch", "trino.plugin.elasticsearch"), new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), new ElasticsearchConnectorModule(), - binder -> { - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }); + new ConnectorContextModule(catalogName, context)); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/client/AwsRequestSigner.java b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/client/AwsRequestSigner.java index ec0f76c88d40..ee7e9f8e98af 100644 --- a/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/client/AwsRequestSigner.java +++ b/plugin/trino-elasticsearch/src/main/java/io/trino/plugin/elasticsearch/client/AwsRequestSigner.java @@ -26,13 +26,13 @@ import org.apache.http.protocol.HttpContext; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.signer.Aws4Signer; -import software.amazon.awssdk.auth.signer.internal.SignerConstant; import software.amazon.awssdk.auth.signer.params.Aws4SignerParams; import software.amazon.awssdk.auth.signer.params.SignerChecksumParams; import software.amazon.awssdk.core.checksums.Algorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.auth.aws.signer.SignerConstant; import software.amazon.awssdk.regions.Region; import java.io.IOException; diff --git a/plugin/trino-elasticsearch/src/test/java/io/trino/plugin/elasticsearch/BaseElasticsearchConnectorTest.java b/plugin/trino-elasticsearch/src/test/java/io/trino/plugin/elasticsearch/BaseElasticsearchConnectorTest.java index 7d5be4a3b307..7a490e51e355 100644 --- a/plugin/trino-elasticsearch/src/test/java/io/trino/plugin/elasticsearch/BaseElasticsearchConnectorTest.java +++ b/plugin/trino-elasticsearch/src/test/java/io/trino/plugin/elasticsearch/BaseElasticsearchConnectorTest.java @@ -18,7 +18,6 @@ import com.google.common.collect.ImmutableMap; import io.trino.Session; import io.trino.spi.type.VarcharType; -import io.trino.sql.planner.plan.LimitNode; import io.trino.testing.AbstractTestQueries; import io.trino.testing.BaseConnectorTest; import io.trino.testing.MaterializedResult; @@ -1711,13 +1710,6 @@ public void testFiltersCharset() .matches("VALUES (VARCHAR 'Türkiye')"); } - @Test - public void testLimitPushdown() - throws IOException - { - assertThat(query("SELECT name FROM nation LIMIT 30")).isNotFullyPushedDown(LimitNode.class); // Use high limit for result determinism - } - @Test public void testDataTypesNested() throws IOException diff --git a/plugin/trino-example-http/pom.xml b/plugin/trino-example-http/pom.xml index 313a75414ce9..a9798d07134e 100644 --- a/plugin/trino-example-http/pom.xml +++ b/plugin/trino-example-http/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-example-http/src/main/java/io/trino/plugin/example/ExampleConnectorFactory.java b/plugin/trino-example-http/src/main/java/io/trino/plugin/example/ExampleConnectorFactory.java index 1622437f4719..0e4faef478f1 100644 --- a/plugin/trino-example-http/src/main/java/io/trino/plugin/example/ExampleConnectorFactory.java +++ b/plugin/trino-example-http/src/main/java/io/trino/plugin/example/ExampleConnectorFactory.java @@ -16,6 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -44,7 +45,8 @@ public Connector create(String catalogName, Map requiredConfig, // A plugin is not required to use Guice; it is just very convenient Bootstrap app = new Bootstrap( new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), + new ConnectorContextModule(catalogName, context), new ExampleModule()); Injector injector = app diff --git a/plugin/trino-example-jdbc/pom.xml b/plugin/trino-example-jdbc/pom.xml index 0c2bddf904d1..31768fc613ec 100644 --- a/plugin/trino-example-jdbc/pom.xml +++ b/plugin/trino-example-jdbc/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-exasol/pom.xml b/plugin/trino-exasol/pom.xml index e0150eed016b..7049d7212f6d 100644 --- a/plugin/trino-exasol/pom.xml +++ b/plugin/trino-exasol/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -113,7 +113,7 @@ com.exasol exasol-testcontainers - 7.1.7 + 7.2.0 test diff --git a/plugin/trino-exchange-filesystem/pom.xml b/plugin/trino-exchange-filesystem/pom.xml index 8443fa3f36fd..2b9d0919b3ae 100644 --- a/plugin/trino-exchange-filesystem/pom.xml +++ b/plugin/trino-exchange-filesystem/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -31,10 +31,6 @@ com.github.stephenc.jcip jcip-annotations - - com.nimbusds - oauth2-oidc-sdk - net.java.dev.jna jna-platform @@ -355,13 +351,6 @@ provided - - com.nimbusds - oauth2-oidc-sdk - jdk11 - runtime - - io.airlift configuration-testing diff --git a/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/FileSystemExchangeManagerFactory.java b/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/FileSystemExchangeManagerFactory.java index 26220496bb0a..4ed38311a5ac 100644 --- a/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/FileSystemExchangeManagerFactory.java +++ b/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/FileSystemExchangeManagerFactory.java @@ -57,6 +57,7 @@ public ExchangeManager create(Map config, ExchangeManagerContext Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/ExchangeS3Config.java b/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/ExchangeS3Config.java index 839506563b84..0d8b93bc288b 100644 --- a/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/ExchangeS3Config.java +++ b/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/ExchangeS3Config.java @@ -310,6 +310,6 @@ public ExchangeS3Config setSseKmsKeyId(String sseKmsKeyId) public boolean isSseConfigValid() { return sseType == KMS && getSseKmsKeyId().isPresent() || - sseType != KMS && getSseKmsKeyId().isEmpty(); + sseType != KMS && getSseKmsKeyId().isEmpty(); } } diff --git a/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/S3FileSystemExchangeStorage.java b/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/S3FileSystemExchangeStorage.java index a78af318c6bb..a3f89930a4ae 100644 --- a/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/S3FileSystemExchangeStorage.java +++ b/plugin/trino-exchange-filesystem/src/main/java/io/trino/plugin/exchange/filesystem/s3/S3FileSystemExchangeStorage.java @@ -56,8 +56,6 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.auth.signer.AwsS3V4Signer; -import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration; import software.amazon.awssdk.awscore.endpoint.AwsClientEndpointProvider; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.checksums.RequestChecksumCalculation; @@ -395,7 +393,6 @@ private ListenableFuture> deleteObjects(String bucke DeleteObjectsRequest request = DeleteObjectsRequest.builder() .bucket(bucketName) .delete(Delete.builder().objects(list.stream().map(key -> ObjectIdentifier.builder().key(key).build()).collect(toImmutableList())).build()) - .overrideConfiguration(disableStrongIntegrityChecksums()) .build(); return toListenableFuture(s3Clients.getUnchecked(bucketName).deleteObjects(request)); }).collect(toImmutableList()))); @@ -927,14 +924,4 @@ private ListenableFuture abortMultipartUpload(Stri return stats.getAbortMultipartUpload().record(toListenableFuture(s3AsyncClient.abortMultipartUpload(abortMultipartUploadRequest))); } } - - // TODO (https://github.com/trinodb/trino/issues/24955): - // remove me once all of the S3-compatible storage support strong integrity checks - @SuppressWarnings("deprecation") - static AwsRequestOverrideConfiguration disableStrongIntegrityChecksums() - { - return AwsRequestOverrideConfiguration.builder() - .signer(AwsS3V4Signer.create()) - .build(); - } } diff --git a/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/alluxio/TestAlluxioFileSystemExchangeManager.java b/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/alluxio/TestAlluxioFileSystemExchangeManager.java index 07f3b1e62f33..64d8c482b4ff 100644 --- a/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/alluxio/TestAlluxioFileSystemExchangeManager.java +++ b/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/alluxio/TestAlluxioFileSystemExchangeManager.java @@ -16,8 +16,8 @@ import io.trino.plugin.exchange.filesystem.AbstractTestExchangeManager; import io.trino.plugin.exchange.filesystem.FileSystemExchangeManagerFactory; import io.trino.plugin.exchange.filesystem.TestExchangeManagerContext; -import io.trino.plugin.exchange.filesystem.containers.AlluxioStorage; import io.trino.spi.exchange.ExchangeManager; +import io.trino.testing.containers.Alluxio; import org.junit.jupiter.api.AfterAll; import static io.trino.plugin.exchange.filesystem.containers.AlluxioStorage.getExchangeManagerProperties; @@ -25,13 +25,13 @@ public class TestAlluxioFileSystemExchangeManager extends AbstractTestExchangeManager { - private AlluxioStorage alluxioStorage; + private Alluxio alluxio; @Override protected ExchangeManager createExchangeManager() { - this.alluxioStorage = new AlluxioStorage(); - alluxioStorage.start(); + this.alluxio = new Alluxio(); + alluxio.start(); return new FileSystemExchangeManagerFactory().create( getExchangeManagerProperties(), @@ -44,9 +44,9 @@ public void destroy() throws Exception { super.destroy(); - if (alluxioStorage != null) { - alluxioStorage.close(); - alluxioStorage = null; + if (alluxio != null) { + alluxio.close(); + alluxio = null; } } } diff --git a/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/containers/AlluxioStorage.java b/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/containers/AlluxioStorage.java index 94ebba571075..fca88735ecb6 100644 --- a/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/containers/AlluxioStorage.java +++ b/plugin/trino-exchange-filesystem/src/test/java/io/trino/plugin/exchange/filesystem/containers/AlluxioStorage.java @@ -14,83 +14,25 @@ package io.trino.plugin.exchange.filesystem.containers; import com.google.common.collect.ImmutableMap; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.utility.DockerImageName; +import io.trino.testing.containers.Alluxio; -import java.time.Duration; import java.util.Map; public class AlluxioStorage implements AutoCloseable { - private static final String IMAGE_NAME = "alluxio/alluxio:2.9.5"; - private static final DockerImageName ALLUXIO_IMAGE = DockerImageName.parse(IMAGE_NAME); - - private GenericContainer alluxioMaster; - private GenericContainer alluxioWorker; + private final Alluxio alluxio = new Alluxio(); public void start() { - alluxioMaster = createAlluxioMasterContainer(); - alluxioWorker = createAlluxioWorkerContainer(); - // the SSHD container will be stopped by TestContainers on shutdown - // https://github.com/trinodb/trino/discussions/21969 - System.setProperty("ReportLeakedContainers.disabled", "true"); - } - - private static GenericContainer createAlluxioMasterContainer() - { - GenericContainer container = new GenericContainer<>(ALLUXIO_IMAGE); - container.withCommand("master-only") - .withEnv("ALLUXIO_JAVA_OPTS", - "-Dalluxio.security.authentication.type=NOSASL " - + "-Dalluxio.master.hostname=localhost " - + "-Dalluxio.worker.hostname=localhost " - + "-Dalluxio.master.mount.table.root.ufs=/opt/alluxio/underFSStorage " - + "-Dalluxio.master.journal.type=NOOP " - + "-Dalluxio.security.authorization.permission.enabled=false " - + "-Dalluxio.security.authorization.plugins.enabled=false ") - .withNetworkMode("host") - .withAccessToHost(true) - .waitingFor(new LogMessageWaitStrategy() - .withRegEx(".*Primary started*\n") - .withStartupTimeout(Duration.ofMinutes(3))); - container.start(); - return container; - } - - private static GenericContainer createAlluxioWorkerContainer() - { - GenericContainer container = new GenericContainer<>(ALLUXIO_IMAGE); - container.withCommand("worker-only") - .withNetworkMode("host") - .withEnv("ALLUXIO_JAVA_OPTS", - "-Dalluxio.security.authentication.type=NOSASL " - + "-Dalluxio.worker.ramdisk.size=512MB " - + "-Dalluxio.worker.hostname=localhost " - + "-Dalluxio.worker.tieredstore.level0.alias=HDD " - + "-Dalluxio.worker.tieredstore.level0.dirs.path=/tmp " - + "-Dalluxio.master.hostname=localhost " - + "-Dalluxio.security.authorization.permission.enabled=false " - + "-Dalluxio.security.authorization.plugins.enabled=false ") - .withAccessToHost(true) - .waitingFor(Wait.forLogMessage(".*Alluxio worker started.*\n", 1)); - container.start(); - return container; + alluxio.start(); } @Override public void close() throws Exception { - if (alluxioMaster != null) { - alluxioMaster.close(); - } - if (alluxioWorker != null) { - alluxioWorker.close(); - } + alluxio.close(); } public static Map getExchangeManagerProperties() diff --git a/plugin/trino-exchange-hdfs/pom.xml b/plugin/trino-exchange-hdfs/pom.xml index 3cd0563229a1..961b699ebd00 100644 --- a/plugin/trino-exchange-hdfs/pom.xml +++ b/plugin/trino-exchange-hdfs/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-exchange-hdfs/src/main/java/io/trino/plugin/exchange/hdfs/HdfsExchangeManagerFactory.java b/plugin/trino-exchange-hdfs/src/main/java/io/trino/plugin/exchange/hdfs/HdfsExchangeManagerFactory.java index 2761a65ef98b..eb7746ddd9b3 100644 --- a/plugin/trino-exchange-hdfs/src/main/java/io/trino/plugin/exchange/hdfs/HdfsExchangeManagerFactory.java +++ b/plugin/trino-exchange-hdfs/src/main/java/io/trino/plugin/exchange/hdfs/HdfsExchangeManagerFactory.java @@ -58,6 +58,7 @@ public ExchangeManager create(Map config, ExchangeManagerContext Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-faker/pom.xml b/plugin/trino-faker/pom.xml index c91d68e78eb0..2cfa8d675836 100644 --- a/plugin/trino-faker/pom.xml +++ b/plugin/trino-faker/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -76,7 +76,7 @@ net.datafaker datafaker - 2.5.1 + 2.5.3 diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java index a4e95895d2df..f7d9120419aa 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerConnectorFactory.java @@ -16,6 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; @@ -48,10 +49,14 @@ public Connector create(String catalogName, Map requiredConfig, requireNonNull(context, "context is null"); checkStrictSpiVersionMatch(context, this); - Bootstrap app = new Bootstrap("io.trino.bootstrap.catalog." + catalogName, new FakerModule(context.getTypeManager())); + Bootstrap app = new Bootstrap( + "io.trino.bootstrap.catalog." + catalogName, + new ConnectorContextModule(catalogName, context), + new FakerModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(requiredConfig) .initialize(); diff --git a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java index c1d5272f8978..e87133b574ff 100644 --- a/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java +++ b/plugin/trino-faker/src/main/java/io/trino/plugin/faker/FakerModule.java @@ -15,30 +15,17 @@ package io.trino.plugin.faker; import com.google.inject.Binder; -import com.google.inject.Inject; import com.google.inject.Module; import com.google.inject.Scopes; -import io.trino.spi.type.TypeManager; import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.util.Objects.requireNonNull; public class FakerModule implements Module { - private final TypeManager typeManager; - - @Inject - public FakerModule(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - @Override public void configure(Binder binder) { - binder.bind(TypeManager.class).toInstance(typeManager); - binder.bind(FakerConnector.class).in(Scopes.SINGLETON); binder.bind(FakerMetadata.class).in(Scopes.SINGLETON); binder.bind(FakerSplitManager.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-functions-python/pom.xml b/plugin/trino-functions-python/pom.xml index b3a828252797..c670ac15fbe0 100644 --- a/plugin/trino-functions-python/pom.xml +++ b/plugin/trino-functions-python/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -18,7 +18,7 @@ com.dylibso.chicory bom - 1.5.1 + 1.5.3 pom import diff --git a/plugin/trino-geospatial/pom.xml b/plugin/trino-geospatial/pom.xml index 988ebca38b79..b6ae983d299b 100644 --- a/plugin/trino-geospatial/pom.xml +++ b/plugin/trino-geospatial/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -232,4 +232,23 @@ test + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + + + diff --git a/plugin/trino-google-sheets/pom.xml b/plugin/trino-google-sheets/pom.xml index 6d8a675b2f33..ce6fba82fa00 100644 --- a/plugin/trino-google-sheets/pom.xml +++ b/plugin/trino-google-sheets/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsClient.java b/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsClient.java index 18ec77dac8f7..9f15afae1827 100644 --- a/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsClient.java +++ b/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsClient.java @@ -295,8 +295,7 @@ private static Credential credentialFromStream(InputStream inputStream, Optional throws IOException { GoogleCredential credential = GoogleCredential.fromStream(inputStream).createScoped(SCOPES); - delegatedUserEmail.ifPresent(credential::createDelegated); - return credential; + return delegatedUserEmail.map(credential::createDelegated).orElse(credential); } private List> readAllValuesFromSheetExpression(String sheetExpression) diff --git a/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsConnectorFactory.java b/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsConnectorFactory.java index 5f5fef375d7c..7a7f5fe522e1 100644 --- a/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsConnectorFactory.java +++ b/plugin/trino-google-sheets/src/main/java/io/trino/plugin/google/sheets/SheetsConnectorFactory.java @@ -16,6 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -44,11 +45,13 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), + new ConnectorContextModule(catalogName, context), new SheetsModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-hive/pom.xml b/plugin/trino-hive/pom.xml index a8de44dc2689..6fad162fbb3a 100644 --- a/plugin/trino-hive/pom.xml +++ b/plugin/trino-hive/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -176,6 +176,11 @@ parquet-column + + org.apache.parquet + parquet-common + + org.apache.parquet parquet-format-structures @@ -330,12 +335,6 @@ runtime - - org.apache.parquet - parquet-common - runtime - - org.jetbrains annotations @@ -423,6 +422,13 @@ test + + io.trino + trino-parquet + test-jar + test + + io.trino trino-parser @@ -555,6 +561,20 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java index 0d4a04dfc9a6..f7f6236ead7c 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/BackgroundHiveSplitLoader.java @@ -52,6 +52,7 @@ import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.Constraint; import io.trino.spi.connector.DynamicFilter; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.type.TypeManager; @@ -148,6 +149,7 @@ public class BackgroundHiveSplitLoader private final Table table; private final TupleDomain compactEffectivePredicate; + private final Constraint constraint; private final DynamicFilter dynamicFilter; private final long dynamicFilteringWaitTimeoutMillis; private final TypeManager typeManager; @@ -192,6 +194,7 @@ public BackgroundHiveSplitLoader( Table table, Iterator partitions, TupleDomain compactEffectivePredicate, + Constraint constraint, DynamicFilter dynamicFilter, Duration dynamicFilteringWaitTimeout, TypeManager typeManager, @@ -209,6 +212,7 @@ public BackgroundHiveSplitLoader( { this.table = table; this.compactEffectivePredicate = compactEffectivePredicate; + this.constraint = constraint; this.dynamicFilter = dynamicFilter; this.dynamicFilteringWaitTimeoutMillis = dynamicFilteringWaitTimeout.toMillis(); this.typeManager = typeManager; @@ -423,6 +427,7 @@ private ListenableFuture loadPartition(HivePartitionMetadata partition) schema, partitionKeys, effectivePredicate, + constraint, partitionMatchSupplier, partition.getHiveColumnCoercions(), Optional.empty(), @@ -475,6 +480,7 @@ private ListenableFuture loadPartition(HivePartitionMetadata partition) schema, partitionKeys, effectivePredicate, + constraint, partitionMatchSupplier, partition.getHiveColumnCoercions(), bucketConversionRequiresWorkerParticipation ? bucketConversion : Optional.empty(), diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java index a16c7551992b..08ca158ec9c4 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveConnectorFactory.java @@ -20,12 +20,10 @@ import io.airlift.bootstrap.Bootstrap; import io.airlift.bootstrap.LifeCycleManager; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.manager.FileSystemModule; import io.trino.metastore.HiveMetastore; -import io.trino.plugin.base.CatalogNameModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorAccessControl; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSinkProvider; @@ -39,12 +37,6 @@ import io.trino.plugin.hive.procedure.HiveProcedureModule; import io.trino.plugin.hive.security.HiveSecurityModule; import io.trino.plugin.hive.security.SystemTableAwareAccessControl; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.PageIndexerFactory; -import io.trino.spi.PageSorter; -import io.trino.spi.VersionEmbedder; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorAccessControl; @@ -54,7 +46,6 @@ import io.trino.spi.connector.ConnectorPageSinkProvider; import io.trino.spi.connector.ConnectorPageSourceProvider; import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.connector.MetadataProvider; import io.trino.spi.connector.TableProcedureMetadata; import io.trino.spi.procedure.Procedure; import org.weakref.jmx.guice.MBeanModule; @@ -80,7 +71,7 @@ public String getName() public Connector create(String catalogName, Map config, ConnectorContext context) { checkStrictSpiVersionMatch(context, this); - return createConnector(catalogName, config, context, EMPTY_MODULE, Optional.empty(), Optional.empty()); + return createConnector(catalogName, config, context, EMPTY_MODULE, Optional.empty(), false, Optional.empty()); } public static Connector createConnector( @@ -89,42 +80,32 @@ public static Connector createConnector( ConnectorContext context, Module module, Optional metastore, + boolean metastoreImpersonationEnabled, Optional fileSystemFactory) { ClassLoader classLoader = HiveConnectorFactory.class.getClassLoader(); try (ThreadContextClassLoader _ = new ThreadContextClassLoader(classLoader)) { Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, - new CatalogNameModule(catalogName), new MBeanModule(), new ConnectorObjectNameGeneratorModule("io.trino.plugin.hive", "trino.plugin.hive"), new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), new HiveModule(), - new HiveMetastoreModule(metastore), + new HiveMetastoreModule(metastore, metastoreImpersonationEnabled), new HiveSecurityModule(), fileSystemFactory .map(factory -> (Module) binder -> binder.bind(TrinoFileSystemFactory.class).toInstance(factory)) - .orElseGet(() -> new FileSystemModule(catalogName, context.getCurrentNode().isCoordinator(), context.getOpenTelemetry(), false)), + .orElseGet(() -> new FileSystemModule(catalogName, context, false)), new HiveProcedureModule(), new MBeanServerModule(), - binder -> { - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getCurrentNode().getVersion())); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(VersionEmbedder.class).toInstance(context.getVersionEmbedder()); - binder.bind(MetadataProvider.class).toInstance(context.getMetadataProvider()); - binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); - binder.bind(PageSorter.class).toInstance(context.getPageSorter()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }, + new ConnectorContextModule(catalogName, context), binder -> newSetBinder(binder, SessionPropertiesProvider.class).addBinding().to(HiveSessionProperties.class).in(Scopes.SINGLETON), module); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java index 3e77c533aed3..dd3a1f168b65 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadata.java @@ -117,6 +117,7 @@ import io.trino.spi.expression.Variable; import io.trino.spi.function.LanguageFunction; import io.trino.spi.function.SchemaFunctionName; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.NullableValue; import io.trino.spi.predicate.TupleDomain; @@ -830,6 +831,13 @@ public Optional getInfo(ConnectorSession session, ConnectorTableHandle t tableDefaultFileFormat)); } + @Override + public Metrics getMetrics(ConnectorSession session) + { + // HiveMetadata and metastore are created per session + return metastore.getMetrics(); + } + @Override public List listTables(ConnectorSession session, Optional optionalSchemaName) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java index 6c2e71331750..31c45bc365a3 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveMetadataFactory.java @@ -27,6 +27,7 @@ import io.trino.plugin.hive.security.AccessControlMetadataFactory; import io.trino.plugin.hive.security.UsingSystemSecurity; import io.trino.plugin.hive.statistics.MetastoreHiveStatisticsProvider; +import io.trino.spi.NodeVersion; import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.MetadataProvider; import io.trino.spi.security.ConnectorIdentity; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java index 46c4affef94a..30c454f9517b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveModule.java @@ -14,14 +14,15 @@ package io.trino.plugin.hive; import com.google.inject.Binder; -import com.google.inject.Module; import com.google.inject.Provides; import com.google.inject.Scopes; import com.google.inject.Singleton; import com.google.inject.multibindings.Multibinder; +import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.avro.AvroFileWriterFactory; import io.trino.plugin.hive.avro.AvroPageSourceFactory; +import io.trino.plugin.hive.crypto.ParquetEncryptionModule; import io.trino.plugin.hive.esri.EsriPageSourceFactory; import io.trino.plugin.hive.fs.CachingDirectoryLister; import io.trino.plugin.hive.fs.DirectoryLister; @@ -61,10 +62,10 @@ import static org.weakref.jmx.guice.ExportBinder.newExporter; public class HiveModule - implements Module + extends AbstractConfigurationAwareModule { @Override - public void configure(Binder binder) + public void setup(Binder binder) { configBinder(binder).bindConfig(HiveConfig.class); configBinder(binder).bindConfig(HiveMetastoreConfig.class); @@ -136,6 +137,7 @@ public void configure(Binder binder) fileWriterFactoryBinder.addBinding().to(ParquetFileWriterFactory.class).in(Scopes.SINGLETON); binder.install(new HiveExecutorModule()); + install(new ParquetEncryptionModule()); } @Provides diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveSplitManager.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveSplitManager.java index 90eaf2faae82..ded5da16445c 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveSplitManager.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/HiveSplitManager.java @@ -263,6 +263,7 @@ public ConnectorSplitSource getSplits( table, hivePartitions, hiveTable.getCompactEffectivePredicate(), + constraint, dynamicFilter, getDynamicFilteringWaitTimeout(session), typeManager, diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriterFactory.java index 3b2e0fbb984c..aa3ba3ce3c59 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/RcFileFileWriterFactory.java @@ -26,6 +26,7 @@ import io.trino.memory.context.AggregatedMemoryContext; import io.trino.metastore.StorageFormat; import io.trino.plugin.hive.acid.AcidTransaction; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.type.Type; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/avro/AvroFileWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/avro/AvroFileWriterFactory.java index 7d81ad3249b4..49e98aa2cc1a 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/avro/AvroFileWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/avro/AvroFileWriterFactory.java @@ -29,9 +29,9 @@ import io.trino.plugin.hive.HiveCompressionCodec; import io.trino.plugin.hive.HiveFileWriterFactory; import io.trino.plugin.hive.HiveTimestampPrecision; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.WriterKind; import io.trino.plugin.hive.acid.AcidTransaction; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.type.Type; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/EnvironmentDecryptionKeyRetriever.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/EnvironmentDecryptionKeyRetriever.java new file mode 100644 index 000000000000..3d1ad85eae75 --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/EnvironmentDecryptionKeyRetriever.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.crypto; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import io.trino.parquet.crypto.DecryptionKeyRetriever; +import org.apache.parquet.hadoop.metadata.ColumnPath; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * Reads keys from two environment variables. + * 
    + *   pme.environment-key-retriever.footer-keys  =  <single‑key> | id1:key1,id2:key2 …
+ *   pme.environment-key-retriever.column-keys  =  <single‑key> | id1:key1,id2:key2 …
+ *
    + *
      + *
    • If the value contains ‘:’ we treat it as a map (comma‑separated {@code id:key}). + * The {@code id} must match the {@code keyMetadata} supplied by Parquet.
      • + *
    • Otherwise it is a single default key, independent of {@code keyMetadata}.
      • + *
    • Keys are expected to be Base‑64; if decoding fails we fall back to the raw UTF‑8 bytes.
      • + *
    + */ +public final class EnvironmentDecryptionKeyRetriever + implements DecryptionKeyRetriever +{ + private static final String FOOTER_VARIABLE_NAME = "pme.environment-key-retriever.footer-keys"; + private static final String COLUMN_VARIABLE_NAME = "pme.environment-key-retriever.column-keys"; + + private final KeySource footerKeys; + private final KeySource columnKeys; + + @Inject + public EnvironmentDecryptionKeyRetriever() + { + this(parseEnvironmentVariable(FOOTER_VARIABLE_NAME), parseEnvironmentVariable(COLUMN_VARIABLE_NAME)); + } + + @VisibleForTesting + EnvironmentDecryptionKeyRetriever(String footerValue, String columnValue) + { + this(parseValue(footerValue, FOOTER_VARIABLE_NAME), parseValue(columnValue, COLUMN_VARIABLE_NAME)); + } + + private EnvironmentDecryptionKeyRetriever(KeySource footerKeys, KeySource columnKeys) + { + this.footerKeys = footerKeys; + this.columnKeys = columnKeys; + } + + @Override + public Optional getColumnKey(ColumnPath columnPath, Optional keyMetadata) + { + return columnKeys.resolve(keyMetadata); + } + + @Override + public Optional getFooterKey(Optional keyMetadata) + { + return footerKeys.resolve(keyMetadata); + } + + private static KeySource parseEnvironmentVariable(String variable) + { + return parseValue(System.getenv(variable), variable); + } + + private static KeySource parseValue(String value, String variable) + { + if (value == null || value.isBlank()) { + return KeySource.empty(); + } + if (value.contains(":")) { + // map mode + ImmutableMap.Builder map = ImmutableMap.builder(); + for (String entry : value.split("\\s*,\\s*")) { + checkArgument(!entry.isBlank(), "Empty entry in %s", variable); + if (entry.isBlank()) { + continue; + } + String[] parts = entry.split(":", 2); + checkArgument(parts.length == 2, "Malformed entry in %s: %s", variable, entry); + map.put(ByteBuffer.wrap(parts[0].getBytes(StandardCharsets.UTF_8)), decodeKey(parts[1])); + } + return new KeySource(Optional.empty(), map.buildOrThrow()); + } + // single key mode + return new KeySource(Optional.of(decodeKey(value)), ImmutableMap.of()); + } + + private static byte[] decodeKey(String token) + { + return Base64.getDecoder().decode(token); + } + + /** + * container for either a single default key or a map keyed by key‑metadata id + */ + private record KeySource(Optional singleKey, Map keyedKeys) + { + static KeySource empty() + { + return new KeySource(Optional.empty(), Collections.emptyMap()); + } + + Optional resolve(Optional keyMetadata) + { + if (singleKey.isPresent()) { + // no map → keyMetadata irrelevant + return singleKey; + } + return keyMetadata.map(bytes -> keyedKeys.get(ByteBuffer.wrap(bytes))); + } + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/ParquetEncryptionConfig.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/ParquetEncryptionConfig.java new file mode 100644 index 000000000000..5126d8daf317 --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/ParquetEncryptionConfig.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.crypto; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; + +import java.util.Optional; + +public class ParquetEncryptionConfig +{ + private boolean environmentKeyRetrieverEnabled; + private Optional aadPrefix = Optional.empty(); + private boolean checkFooterIntegrity = true; + + @Config("pme.environment-key-retriever.enabled") + @ConfigDescription("Enable the key retriever that retrieves keys from the environment variable") + public ParquetEncryptionConfig setEnvironmentKeyRetrieverEnabled(boolean enabled) + { + this.environmentKeyRetrieverEnabled = enabled; + return this; + } + + public boolean isEnvironmentKeyRetrieverEnabled() + { + return environmentKeyRetrieverEnabled; + } + + @Config("pme.aad-prefix") + @ConfigDescription("AAD prefix used to decode Parquet files") + public ParquetEncryptionConfig setAadPrefix(String prefix) + { + this.aadPrefix = Optional.ofNullable(prefix); + return this; + } + + public Optional getAadPrefix() + { + return aadPrefix; + } + + @Config("pme.check-footer-integrity") + @ConfigDescription("Validate signature for plaintext footer files") + public ParquetEncryptionConfig setCheckFooterIntegrity(boolean checkFooterIntegrity) + { + this.checkFooterIntegrity = checkFooterIntegrity; + return this; + } + + public boolean isCheckFooterIntegrity() + { + return checkFooterIntegrity; + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/ParquetEncryptionModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/ParquetEncryptionModule.java new file mode 100644 index 000000000000..c16bad2c38c5 --- /dev/null +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/crypto/ParquetEncryptionModule.java @@ -0,0 +1,98 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.crypto; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import com.google.inject.multibindings.Multibinder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.parquet.crypto.DecryptionKeyRetriever; +import io.trino.parquet.crypto.FileDecryptionProperties; +import org.apache.parquet.hadoop.metadata.ColumnPath; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +public class ParquetEncryptionModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + ParquetEncryptionConfig config = buildConfigObject(ParquetEncryptionConfig.class); + Multibinder retrieverBinder = + Multibinder.newSetBinder(binder, DecryptionKeyRetriever.class); + if (config.isEnvironmentKeyRetrieverEnabled()) { + retrieverBinder.addBinding().to(EnvironmentDecryptionKeyRetriever.class).in(Scopes.SINGLETON); + } + } + + @Provides + @Singleton + public Optional fileDecryptionProperties( + ParquetEncryptionConfig config, + Set retrievers) + { + if (retrievers.isEmpty()) { + return Optional.empty(); + } + + DecryptionKeyRetriever aggregate = new CompositeDecryptionKeyRetriever(List.copyOf(retrievers)); + + FileDecryptionProperties.Builder builder = FileDecryptionProperties.builder() + .withKeyRetriever(aggregate) + .withCheckFooterIntegrity(config.isCheckFooterIntegrity()); + + config.getAadPrefix() + .map(string -> string.getBytes(StandardCharsets.UTF_8)) + .ifPresent(builder::withAadPrefix); + + return Optional.of(builder.build()); + } + + private static class CompositeDecryptionKeyRetriever + implements DecryptionKeyRetriever + { + private final List delegates; + + CompositeDecryptionKeyRetriever(List delegates) + { + this.delegates = List.copyOf(delegates); + } + + @Override + public Optional getColumnKey(ColumnPath path, Optional meta) + { + return delegates.stream() + .map(delegate -> delegate.getColumnKey(path, meta)) + .filter(Optional::isPresent) + .findFirst() + .orElse(Optional.empty()); + } + + @Override + public Optional getFooterKey(Optional meta) + { + return delegates.stream() + .map(delegate -> delegate.getFooterKey(meta)) + .filter(Optional::isPresent) + .findFirst() + .orElse(Optional.empty()); + } + } +} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/line/SimpleSequenceFileWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/line/SimpleSequenceFileWriterFactory.java index f1aa4bd4883f..19a0a91ad59b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/line/SimpleSequenceFileWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/line/SimpleSequenceFileWriterFactory.java @@ -17,7 +17,7 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.hive.formats.line.sequence.SequenceFileWriterFactory; import io.trino.hive.formats.line.simple.SimpleSerializerFactory; -import io.trino.plugin.hive.NodeVersion; +import io.trino.spi.NodeVersion; import io.trino.spi.type.TypeManager; import static io.trino.hive.formats.line.sequence.ValueType.TEXT; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java index 63e94f7d6860..570dc5ad7389 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/CachingHiveMetastoreModule.java @@ -19,6 +19,7 @@ import com.google.inject.Singleton; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.metastore.MeasuredHiveMetastore.MeasuredMetastoreFactory; import io.trino.metastore.RawHiveMetastoreFactory; import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.metastore.cache.CachingHiveMetastoreConfig; @@ -65,7 +66,7 @@ public static HiveMetastoreFactory createHiveMetastore( SharedHiveMetastoreCache sharedHiveMetastoreCache) { // cross TX metastore cache is enabled wrapper with caching metastore - return sharedHiveMetastoreCache.createCachingHiveMetastoreFactory(metastoreFactory); + return sharedHiveMetastoreCache.createCachingHiveMetastoreFactory(new MeasuredMetastoreFactory(metastoreFactory)); } @Provides diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java index f069dec6f132..d27d9503188d 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java @@ -30,17 +30,19 @@ public class HiveMetastoreModule extends AbstractConfigurationAwareModule { private final Optional metastore; + private final boolean impersonationEnabled; - public HiveMetastoreModule(Optional metastore) + public HiveMetastoreModule(Optional metastore, boolean impersonationEnabled) { this.metastore = metastore; + this.impersonationEnabled = impersonationEnabled; } @Override protected void setup(Binder binder) { if (metastore.isPresent()) { - binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore.get())); + binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore.get(), impersonationEnabled)); binder.bind(Key.get(boolean.class, AllowHiveTableRename.class)).toInstance(true); } else { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java index f8f29e900ae9..31ef18e7c9a8 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/SemiTransactionalHiveMetastore.java @@ -68,6 +68,7 @@ import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.function.LanguageFunction; import io.trino.spi.function.SchemaFunctionName; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.TupleDomain; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.security.PrincipalType; @@ -1282,6 +1283,11 @@ public void checkSupportsHiveAcidTransactions() delegate.checkSupportsTransactions(); } + public Metrics getMetrics() + { + return delegate.getMetrics(); + } + public void beginQuery(ConnectorSession session) { String queryId = session.getQueryId(); diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java index 9ec3ee2c3fd4..5e63798f78fa 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastore.java @@ -44,10 +44,10 @@ import io.trino.metastore.Table; import io.trino.metastore.TableAlreadyExistsException; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.PartitionNotFoundException; import io.trino.plugin.hive.TableType; import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig.VersionCompatibility; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnNotFoundException; import io.trino.spi.connector.SchemaNotFoundException; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java index 6d1a9101c7a4..cfc450ef6fca 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/file/FileHiveMetastoreFactory.java @@ -20,7 +20,7 @@ import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.tracing.TracingHiveMetastore; import io.trino.plugin.hive.HideDeltaLakeTables; -import io.trino.plugin.hive.NodeVersion; +import io.trino.spi.NodeVersion; import io.trino.spi.security.ConnectorIdentity; import java.util.Optional; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java index af8b4af0db07..95ef5a306bb5 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/thrift/ThriftHiveMetastoreClient.java @@ -754,7 +754,7 @@ public void alterTransactionalTable(Table table, long transactionId, long writeI }, () -> { table.setWriteId(originalWriteId); - client.alterTableWithEnvironmentContext(table.getDbName(), table.getTableName(), table, environmentContext); + client.alterTableWithEnvironmentContext(prependCatalogToDbName(catalogName, table.getDbName()), table.getTableName(), table, environmentContext); return null; }); } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java index 148335703b5e..322056313178 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/orc/OrcFileWriterFactory.java @@ -31,9 +31,9 @@ import io.trino.plugin.hive.FileWriter; import io.trino.plugin.hive.HiveCompressionCodec; import io.trino.plugin.hive.HiveFileWriterFactory; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.WriterKind; import io.trino.plugin.hive.acid.AcidTransaction; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.type.Type; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriterFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriterFactory.java index bc3f944b6f9b..f1be32b946ac 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriterFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetFileWriterFactory.java @@ -29,9 +29,9 @@ import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.HiveFileWriterFactory; import io.trino.plugin.hive.HiveSessionProperties; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.WriterKind; import io.trino.plugin.hive.acid.AcidTransaction; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.type.Type; diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java index a8df03061c89..33c144f2e12b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/parquet/ParquetPageSourceFactory.java @@ -33,6 +33,7 @@ import io.trino.parquet.ParquetDataSourceId; import io.trino.parquet.ParquetReaderOptions; import io.trino.parquet.ParquetWriteValidation; +import io.trino.parquet.crypto.FileDecryptionProperties; import io.trino.parquet.metadata.FileMetadata; import io.trino.parquet.metadata.ParquetMetadata; import io.trino.parquet.predicate.TupleDomainParquetPredicate; @@ -134,6 +135,7 @@ public class ParquetPageSourceFactory private final TrinoFileSystemFactory fileSystemFactory; private final FileFormatDataSourceStats stats; + private final Optional fileDecryptionProperties; private final ParquetReaderOptions options; private final DateTimeZone timeZone; private final int domainCompactionThreshold; @@ -142,11 +144,13 @@ public class ParquetPageSourceFactory public ParquetPageSourceFactory( TrinoFileSystemFactory fileSystemFactory, FileFormatDataSourceStats stats, + Optional fileDecryptionProperties, ParquetReaderConfig config, HiveConfig hiveConfig) { this.fileSystemFactory = requireNonNull(fileSystemFactory, "fileSystemFactory is null"); this.stats = requireNonNull(stats, "stats is null"); + this.fileDecryptionProperties = requireNonNull(fileDecryptionProperties, "fileDecryptionProperties is null"); options = config.toParquetReaderOptions(); timeZone = hiveConfig.getParquetDateTimeZone(); domainCompactionThreshold = hiveConfig.getDomainCompactionThreshold(); @@ -201,6 +205,7 @@ public Optional createPageSource( .withVectorizedDecodingEnabled(isParquetVectorizedDecodingEnabled(session)) .build(), Optional.empty(), + fileDecryptionProperties, domainCompactionThreshold, OptionalLong.of(estimatedFileSize))); } @@ -219,6 +224,7 @@ public static ConnectorPageSource createPageSource( FileFormatDataSourceStats stats, ParquetReaderOptions options, Optional parquetWriteValidation, + Optional fileDecryptionProperties, int domainCompactionThreshold, OptionalLong estimatedFileSize) { @@ -230,7 +236,7 @@ public static ConnectorPageSource createPageSource( AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext(); dataSource = createDataSource(inputFile, estimatedFileSize, options, memoryContext, stats); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.of(options.getMaxFooterReadSize()), parquetWriteValidation); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.of(options.getMaxFooterReadSize()), parquetWriteValidation, fileDecryptionProperties); FileMetadata fileMetaData = parquetMetadata.getFileMetaData(); fileSchema = fileMetaData.getSchema(); @@ -285,7 +291,8 @@ public static ConnectorPageSource createPageSource( // We avoid using disjuncts of parquetPredicate for page pruning in ParquetReader as currently column indexes // are not present in the Parquet files which are read with disjunct predicates. parquetPredicates.size() == 1 ? Optional.of(parquetPredicates.getFirst()) : Optional.empty(), - parquetWriteValidation); + parquetWriteValidation, + parquetMetadata.getDecryptionContext()); return createParquetPageSource(columns, fileSchema, messageColumn, useColumnNames, parquetReaderProvider); } catch (Exception e) { diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java index fffea2dcd713..6f653829ad47 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/procedure/FlushMetadataCacheProcedure.java @@ -21,6 +21,7 @@ import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.Table; import io.trino.metastore.cache.CachingHiveMetastore; +import io.trino.metastore.cache.SharedHiveMetastoreCache.ImpersonationCachingHiveMetastoreFactory; import io.trino.plugin.hive.HiveErrorCode; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.metastore.glue.GlueCache; @@ -133,6 +134,18 @@ public void flushMetadataCache( } private void doFlushMetadataCache(ConnectorSession session, Optional schemaName, Optional tableName, List partitionColumns, List partitionValues) + { + if (hiveMetadataFactory instanceof ImpersonationCachingHiveMetastoreFactory impersonationCachingHiveMetastoreFactory) { + checkState(cachingHiveMetastore.isEmpty(), "CachingHiveMetastore should not be set when using ImpersonationCachingHiveMetastoreFactory"); + Optional impersonationCachingHiveMetastore = Optional.of((CachingHiveMetastore) impersonationCachingHiveMetastoreFactory.createMetastore(Optional.of(session.getIdentity()))); + doFlushMetadataCache(session, impersonationCachingHiveMetastore, schemaName, tableName, partitionColumns, partitionValues); + } + else { + doFlushMetadataCache(session, cachingHiveMetastore, schemaName, tableName, partitionColumns, partitionValues); + } + } + + private void doFlushMetadataCache(ConnectorSession session, Optional cachingHiveMetastore, Optional schemaName, Optional tableName, List partitionColumns, List partitionValues) { if (cachingHiveMetastore.isEmpty() && glueCache.isEmpty()) { // TODO this currently does not work. CachingHiveMetastore is always bound for metastores other than Glue, even when caching is disabled, @@ -156,13 +169,13 @@ else if (schemaName.isPresent() && tableName.isPresent()) { List partitions; if (!partitionColumns.isEmpty()) { - cachingHiveMetastore.ifPresent(cachingHiveMetastore -> cachingHiveMetastore.flushPartitionCache(schemaName.get(), tableName.get(), partitionColumns, partitionValues)); + cachingHiveMetastore.ifPresent(hiveMetastore -> hiveMetastore.flushPartitionCache(schemaName.get(), tableName.get(), partitionColumns, partitionValues)); glueCache.ifPresent(glueCache -> glueCache.invalidatePartition(schemaName.get(), tableName.get(), new PartitionName(partitionValues))); partitions = ImmutableList.of(makePartName(partitionColumns, partitionValues)); } else { - cachingHiveMetastore.ifPresent(cachingHiveMetastore -> cachingHiveMetastore.invalidateTable(schemaName.get(), tableName.get())); + cachingHiveMetastore.ifPresent(hiveMetastore -> hiveMetastore.invalidateTable(schemaName.get(), tableName.get())); glueCache.ifPresent(glueCache -> glueCache.invalidateTable(schemaName.get(), tableName.get(), true)); List partitionColumnNames = table.getPartitionColumns().stream() diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/BlockJsonSerde.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/BlockJsonSerde.java deleted file mode 100644 index 4a3423f8f98c..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/BlockJsonSerde.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.util; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.google.inject.Inject; -import io.airlift.slice.BasicSliceInput; -import io.airlift.slice.DynamicSliceOutput; -import io.airlift.slice.SliceOutput; -import io.airlift.slice.Slices; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockEncodingSerde; - -import java.io.IOException; -import java.util.Base64; - -import static java.util.Objects.requireNonNull; - -// copy of TestingBlockJsonSerde -public final class BlockJsonSerde -{ - private BlockJsonSerde() {} - - public static class Serializer - extends JsonSerializer - { - private final BlockEncodingSerde blockEncodingSerde; - - @Inject - public Serializer(HiveBlockEncodingSerde blockEncodingSerde) - { - this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); - } - - @Override - public void serialize(Block block, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) - throws IOException - { - SliceOutput output = new DynamicSliceOutput(64); - blockEncodingSerde.writeBlock(output, block); - String encoded = Base64.getEncoder().encodeToString(output.slice().getBytes()); - jsonGenerator.writeString(encoded); - } - } - - public static class Deserializer - extends JsonDeserializer - { - private final BlockEncodingSerde blockEncodingSerde; - - @Inject - public Deserializer(HiveBlockEncodingSerde blockEncodingSerde) - { - this.blockEncodingSerde = requireNonNull(blockEncodingSerde, "blockEncodingSerde is null"); - } - - @Override - public Block deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) - throws IOException - { - byte[] decoded = Base64.getDecoder().decode(jsonParser.readValueAs(String.class)); - BasicSliceInput input = Slices.wrappedBuffer(decoded).getInput(); - return blockEncodingSerde.readBlock(input); - } - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveBlockEncodingSerde.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveBlockEncodingSerde.java deleted file mode 100644 index 00afd090724a..000000000000 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/HiveBlockEncodingSerde.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.util; - -import com.google.common.collect.ImmutableList; -import com.google.inject.Inject; -import io.airlift.slice.SliceInput; -import io.airlift.slice.SliceOutput; -import io.trino.spi.block.ArrayBlockEncoding; -import io.trino.spi.block.Block; -import io.trino.spi.block.BlockEncoding; -import io.trino.spi.block.BlockEncodingSerde; -import io.trino.spi.block.ByteArrayBlockEncoding; -import io.trino.spi.block.DictionaryBlockEncoding; -import io.trino.spi.block.Fixed12BlockEncoding; -import io.trino.spi.block.Int128ArrayBlockEncoding; -import io.trino.spi.block.IntArrayBlockEncoding; -import io.trino.spi.block.LongArrayBlockEncoding; -import io.trino.spi.block.MapBlockEncoding; -import io.trino.spi.block.RowBlockEncoding; -import io.trino.spi.block.RunLengthBlockEncoding; -import io.trino.spi.block.ShortArrayBlockEncoding; -import io.trino.spi.block.VariableWidthBlockEncoding; -import io.trino.spi.type.Type; -import io.trino.spi.type.TypeId; - -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import static com.google.common.base.Preconditions.checkArgument; -import static io.airlift.slice.SizeOf.SIZE_OF_INT; -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.HyperLogLogType.HYPER_LOG_LOG; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; -import static io.trino.spi.type.TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS; -import static io.trino.spi.type.VarbinaryType.VARBINARY; -import static io.trino.spi.type.VarcharType.VARCHAR; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; - -// This class is exactly the same as BlockEncodingManager -public final class HiveBlockEncodingSerde - implements BlockEncodingSerde -{ - private static final List TYPES = ImmutableList.of(BOOLEAN, BIGINT, DOUBLE, INTEGER, VARCHAR, VARBINARY, TIMESTAMP_MILLIS, TIMESTAMP_TZ_MILLIS, DATE, HYPER_LOG_LOG); - - private final ConcurrentMap blockEncodingsByName = new ConcurrentHashMap<>(); - private final ConcurrentMap, BlockEncoding> blockEncodingsByClass = new ConcurrentHashMap<>(); - - @Inject - public HiveBlockEncodingSerde() - { - addBlockEncoding(new VariableWidthBlockEncoding()); - addBlockEncoding(new ByteArrayBlockEncoding()); - addBlockEncoding(new ShortArrayBlockEncoding()); - addBlockEncoding(new IntArrayBlockEncoding()); - addBlockEncoding(new LongArrayBlockEncoding()); - addBlockEncoding(new Fixed12BlockEncoding()); - addBlockEncoding(new Int128ArrayBlockEncoding()); - addBlockEncoding(new DictionaryBlockEncoding()); - addBlockEncoding(new ArrayBlockEncoding()); - addBlockEncoding(new MapBlockEncoding()); - addBlockEncoding(new RowBlockEncoding()); - addBlockEncoding(new RunLengthBlockEncoding()); - } - - private void addBlockEncoding(BlockEncoding blockEncoding) - { - blockEncodingsByClass.put(blockEncoding.getBlockClass(), blockEncoding); - blockEncodingsByName.put(blockEncoding.getName(), blockEncoding); - } - - @Override - public Block readBlock(SliceInput input) - { - // read the encoding name - String encodingName = readLengthPrefixedString(input); - - // look up the encoding factory - BlockEncoding blockEncoding = blockEncodingsByName.get(encodingName); - checkArgument(blockEncoding != null, "Unknown block encoding %s", encodingName); - - // load read the encoding factory from the output stream - return blockEncoding.readBlock(this, input); - } - - @Override - public long estimatedWriteSize(Block block) - { - while (true) { - BlockEncoding blockEncoding = blockEncodingsByClass.get(block.getClass()); - // see if a replacement block should be written instead - Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); - if (replacementBlock.isPresent()) { - block = replacementBlock.get(); - continue; - } - // length of encoding name + encoding name + block size - // TODO: improve this estimate by adding estimatedWriteSize to BlockEncoding interface - return SIZE_OF_INT + blockEncoding.getName().length() + block.getSizeInBytes(); - } - } - - @Override - public void writeBlock(SliceOutput output, Block block) - { - while (true) { - // look up the encoding factory - BlockEncoding blockEncoding = blockEncodingsByClass.get(block.getClass()); - checkArgument(blockEncoding != null, "Cannot write block %s as no encoding was found", block); - - // see if a replacement block should be written instead - Optional replacementBlock = blockEncoding.replacementBlockForWrite(block); - if (replacementBlock.isPresent()) { - block = replacementBlock.get(); - continue; - } - - // write the name to the output - writeLengthPrefixedString(output, blockEncoding.getName()); - - // write the block to the output - blockEncoding.writeBlock(this, output, block); - - break; - } - } - - @Override - public Type readType(SliceInput sliceInput) - { - requireNonNull(sliceInput, "sliceInput is null"); - - TypeId id = TypeId.of(readLengthPrefixedString(sliceInput)); - for (Type type : TYPES) { - if (type.getTypeId().equals(id)) { - return type; - } - } - throw new IllegalArgumentException("Type not found: " + id); - } - - @Override - public void writeType(SliceOutput sliceOutput, Type type) - { - requireNonNull(sliceOutput, "sliceOutput is null"); - requireNonNull(type, "type is null"); - writeLengthPrefixedString(sliceOutput, type.getTypeId().getId()); - } - - private static String readLengthPrefixedString(SliceInput input) - { - int length = input.readInt(); - byte[] bytes = new byte[length]; - input.readBytes(bytes); - return new String(bytes, UTF_8); - } - - private static void writeLengthPrefixedString(SliceOutput output, String value) - { - byte[] bytes = value.getBytes(UTF_8); - output.writeInt(bytes.length); - output.writeBytes(bytes); - } -} diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/InternalHiveSplitFactory.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/InternalHiveSplitFactory.java index 241454233f0e..2b2d3636184b 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/InternalHiveSplitFactory.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/InternalHiveSplitFactory.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.airlift.slice.Slice; import io.airlift.units.DataSize; import io.trino.metastore.HiveTypeName; import io.trino.plugin.hive.AcidInfo; @@ -32,7 +33,10 @@ import io.trino.plugin.hive.parquet.ParquetPageSourceFactory; import io.trino.plugin.hive.rcfile.RcFilePageSourceFactory; import io.trino.spi.HostAddress; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.Constraint; import io.trino.spi.predicate.Domain; +import io.trino.spi.predicate.NullableValue; import io.trino.spi.predicate.TupleDomain; import java.util.Collection; @@ -41,11 +45,14 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.function.BooleanSupplier; +import java.util.function.Predicate; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.plugin.hive.HiveColumnHandle.PATH_TYPE; import static io.trino.plugin.hive.HiveColumnHandle.isPathColumnHandle; +import static io.trino.plugin.hive.HiveColumnHandle.pathColumnHandle; import static io.trino.plugin.hive.util.AcidTables.isFullAcidTable; import static io.trino.plugin.hive.util.HiveUtil.getSerializationLibraryName; import static java.util.Objects.requireNonNull; @@ -58,6 +65,7 @@ public class InternalHiveSplitFactory private final List partitionKeys; private final Optional pathDomain; private final Map hiveColumnCoercions; + private final Constraint constraint; private final BooleanSupplier partitionMatchSupplier; private final Optional bucketConversion; private final Optional bucketValidation; @@ -71,6 +79,7 @@ public InternalHiveSplitFactory( Map schema, List partitionKeys, TupleDomain effectivePredicate, + Constraint constraint, BooleanSupplier partitionMatchSupplier, Map hiveColumnCoercions, Optional bucketConversion, @@ -84,6 +93,7 @@ public InternalHiveSplitFactory( this.strippedSchema = stripUnnecessaryProperties(requireNonNull(schema, "schema is null")); this.partitionKeys = requireNonNull(partitionKeys, "partitionKeys is null"); pathDomain = getPathDomain(requireNonNull(effectivePredicate, "effectivePredicate is null")); + this.constraint = requireNonNull(constraint, "constraint is null"); this.partitionMatchSupplier = requireNonNull(partitionMatchSupplier, "partitionMatchSupplier is null"); this.hiveColumnCoercions = ImmutableMap.copyOf(requireNonNull(hiveColumnCoercions, "hiveColumnCoercions is null")); this.bucketConversion = requireNonNull(bucketConversion, "bucketConversion is null"); @@ -144,7 +154,7 @@ private Optional createInternalHiveSplit( boolean splittable, Optional acidInfo) { - if (!pathMatchesPredicate(pathDomain, path)) { + if (!pathMatchesPredicate(pathDomain, constraint, path)) { return Optional.empty(); } @@ -259,10 +269,16 @@ private static Optional getPathDomain(TupleDomain effe .findFirst()); } - private static boolean pathMatchesPredicate(Optional pathDomain, String path) + private static boolean pathMatchesPredicate(Optional pathDomain, Constraint constraint, String path) { - return pathDomain - .map(domain -> domain.includesNullableValue(utf8Slice(path))) - .orElse(true); + Slice pathSlice = utf8Slice(path); + if (pathDomain.isPresent() && !pathDomain.get().includesNullableValue(pathSlice)) { + return false; + } + if (constraint.getPredicateColumns().isEmpty() || !constraint.getPredicateColumns().get().contains(pathColumnHandle())) { + return true; + } + Predicate> predicate = constraint.predicate().orElse(_ -> true); + return predicate.test(ImmutableMap.of(pathColumnHandle(), new NullableValue(PATH_TYPE, pathSlice))); } } diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/SortBuffer.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/SortBuffer.java index 06d36e88ae3e..8f4a34e3c9ed 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/SortBuffer.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/util/SortBuffer.java @@ -24,12 +24,12 @@ import io.trino.spi.type.Type; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.function.Consumer; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; -import static com.google.common.base.Verify.verify; import static io.airlift.slice.SizeOf.instanceSize; import static java.lang.Math.addExact; import static java.util.Objects.requireNonNull; @@ -44,7 +44,6 @@ public class SortBuffer private final List sortOrders; private final PageSorter pageSorter; private final List pages = new ArrayList<>(); - private final PageBuilder pageBuilder; private long usedMemoryBytes; private int rowCount; @@ -62,7 +61,6 @@ public SortBuffer( this.sortFields = ImmutableList.copyOf(requireNonNull(sortFields, "sortFields is null")); this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null")); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); - this.pageBuilder = new PageBuilder(types); } public long getRetainedBytes() @@ -103,32 +101,8 @@ public void flushTo(Consumer consumer) { checkState(!pages.isEmpty(), "page buffer is empty"); - long[] addresses = pageSorter.sort(types, pages, sortFields, sortOrders, rowCount); - - int[] pageIndex = new int[addresses.length]; - int[] positionIndex = new int[addresses.length]; - for (int i = 0; i < addresses.length; i++) { - pageIndex[i] = pageSorter.decodePageIndex(addresses[i]); - positionIndex[i] = pageSorter.decodePositionIndex(addresses[i]); - } - - verify(pageBuilder.isEmpty()); - - for (int i = 0; i < pageIndex.length; i++) { - Page page = pages.get(pageIndex[i]); - int position = positionIndex[i]; - appendPositionTo(page, position, pageBuilder); - - if (pageBuilder.isFull()) { - consumer.accept(pageBuilder.build()); - pageBuilder.reset(); - } - } - - if (!pageBuilder.isEmpty()) { - consumer.accept(pageBuilder.build()); - pageBuilder.reset(); - } + Iterator sortedPages = pageSorter.sort(types, pages, sortFields, sortOrders, rowCount); + sortedPages.forEachRemaining(consumer); pages.clear(); rowCount = 0; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java index 7bc854803792..5ddfcde36773 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/BaseHiveConnectorTest.java @@ -22,6 +22,8 @@ import io.airlift.json.ObjectMapperProvider; import io.airlift.units.DataSize; import io.trino.Session; +import io.trino.connector.MockConnectorFactory; +import io.trino.connector.MockConnectorPlugin; import io.trino.execution.QueryInfo; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystem; @@ -40,9 +42,11 @@ import io.trino.spi.connector.CatalogSchemaTableName; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.Constraint; +import io.trino.spi.metrics.Metrics; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.security.Identity; import io.trino.spi.security.SelectedRole; +import io.trino.spi.statistics.TableStatistics; import io.trino.spi.type.DateType; import io.trino.spi.type.TimestampType; import io.trino.spi.type.Type; @@ -64,11 +68,13 @@ import io.trino.testing.QueryRunner; import io.trino.testing.QueryRunner.MaterializedResultWithPlan; import io.trino.testing.TestingConnectorBehavior; +import io.trino.testing.TestingSession; import io.trino.testing.sql.TestTable; import io.trino.testing.sql.TrinoSqlExecutor; import io.trino.type.TypeDeserializer; import org.assertj.core.api.AbstractLongAssert; import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -119,6 +125,7 @@ import static io.trino.SystemSessionProperties.FAULT_TOLERANT_EXECUTION_ARBITRARY_DISTRIBUTION_WRITE_TASK_TARGET_SIZE_MIN; import static io.trino.SystemSessionProperties.FAULT_TOLERANT_EXECUTION_HASH_DISTRIBUTION_COMPUTE_TASK_TARGET_SIZE; import static io.trino.SystemSessionProperties.FAULT_TOLERANT_EXECUTION_HASH_DISTRIBUTION_WRITE_TASK_TARGET_SIZE; +import static io.trino.SystemSessionProperties.ITERATIVE_OPTIMIZER_TIMEOUT; import static io.trino.SystemSessionProperties.MAX_WRITER_TASK_COUNT; import static io.trino.SystemSessionProperties.QUERY_MAX_MEMORY_PER_NODE; import static io.trino.SystemSessionProperties.REDISTRIBUTE_WRITES; @@ -197,6 +204,7 @@ import static java.util.stream.Collectors.toSet; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Fail.fail; import static org.junit.jupiter.api.Assumptions.abort; public abstract class BaseHiveConnectorTest @@ -245,6 +253,29 @@ protected static QueryRunner createHiveQueryRunner(HiveQueryRunner.Builder bu return queryRunner; } + @BeforeAll + public void initMockMetricsCatalog() + { + QueryRunner queryRunner = getQueryRunner(); + String mockConnector = "mock_metrics"; + queryRunner.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder() + .withName(mockConnector) + .withListSchemaNames(_ -> ImmutableList.of("default")) + .withGetTableStatistics(_ -> { + try { + Thread.sleep(110); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + return TableStatistics.empty(); + }) + .build())); + + queryRunner.createCatalog("mock_metrics", mockConnector); + } + @Override protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) { @@ -256,6 +287,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_MATERIALIZED_VIEW, SUPPORTS_DEFAULT_COLUMN_VALUE, SUPPORTS_DROP_FIELD, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MERGE, SUPPORTS_NOT_NULL_CONSTRAINT, SUPPORTS_REFRESH_VIEW, @@ -5599,6 +5631,33 @@ public void testBucketFilteringByInPredicate() } } + @Test + void testPathConstraintSplitPruning() + { + String tableName = "test_path_constraint_split_pruning_" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + " (id bigint, data varchar)"); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'a'), (2, 'b'), (3, 'c')", 3); + assertUpdate("INSERT INTO " + tableName + " VALUES (1, 'a'), (2, 'b'), (3, 'c')", 3); + + try { + Set files = getTableFiles(tableName); + assertThat(files).hasSizeGreaterThanOrEqualTo(2); + String fileName = files.stream().findAny().orElseThrow(); + String query = "SELECT * FROM " + tableName + " WHERE \"$path\" LIKE '%" + fileName + "%'"; + assertQueryStats( + getSession(), + query, + queryStats -> { + assertThat(queryStats.getTotalDrivers()).isEqualTo(1); + assertThat(queryStats.getPhysicalInputPositions()).isEqualTo(3); + }, + results -> assertThat(results.getRowCount()).isEqualTo(3)); + } + finally { + assertUpdate("DROP TABLE " + tableName); + } + } + @Test public void testSchemaMismatchesWithDereferenceProjections() { @@ -9398,6 +9457,39 @@ public void testFlushMetadataDisabled() assertQuerySucceeds("CALL system.flush_metadata_cache()"); } + @Test + public void testCatalogMetadataMetrics() + { + MaterializedResultWithPlan result = getQueryRunner().executeWithPlan( + getSession(), + "SELECT count(*) FROM region r, nation n WHERE r.regionkey = n.regionkey"); + Map metrics = getCatalogMetadataMetrics(result.queryId()); + assertCountMetricExists(metrics, "hive", "metastore.all.time.total"); + assertDistributionMetricExists(metrics, "hive", "metastore.all.time.distribution"); + assertCountMetricExists(metrics, "hive", "metastore.getTable.time.total"); + assertDistributionMetricExists(metrics, "hive", "metastore.getTable.time.distribution"); + assertCountMetricExists(metrics, "hive", "metastore.getTableColumnStatistics.time.total"); + } + + @Test + public void testCatalogMetadataMetricsWithOptimizerTimeoutExceeded() + { + String query = "SELECT count(*) FROM region r, nation n, mock_metrics.default.mock_table m WHERE r.regionkey = n.regionkey"; + try { + Session smallOptimizerTimeout = TestingSession.testSessionBuilder(getSession()) + .setSystemProperty(ITERATIVE_OPTIMIZER_TIMEOUT, "100ms") + .build(); + MaterializedResultWithPlan result = getQueryRunner().executeWithPlan(smallOptimizerTimeout, query); + fail(format("Expected query to fail: %s [QueryId: %s]", query, result.queryId())); + } + catch (QueryFailedException e) { + assertThat(e.getMessage()).contains("The optimizer exhausted the time limit"); + Map metrics = getCatalogMetadataMetrics(e.getQueryId()); + assertCountMetricExists(metrics, "hive", "metastore.all.time.total"); + assertCountMetricExists(metrics, "hive", "metastore.getTable.time.total"); + } + } + private static final Set NAMED_COLUMN_ONLY_FORMATS = ImmutableSet.of(HiveStorageFormat.AVRO, HiveStorageFormat.JSON); private Session getParallelWriteSession(Session baseSession) diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java index 2eb36d67a8e5..595c978633db 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveQueryRunner.java @@ -23,6 +23,7 @@ import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; import io.trino.metastore.HiveMetastoreFactory; +import io.trino.parquet.crypto.DecryptionKeyRetriever; import io.trino.plugin.tpcds.TpcdsPlugin; import io.trino.plugin.tpch.ColumnNaming; import io.trino.plugin.tpch.DecimalTypeMapping; @@ -46,6 +47,7 @@ import static io.airlift.log.Level.WARN; import static io.airlift.units.Duration.nanosSince; +import static io.trino.plugin.hive.HiveTestUtils.SESSION; import static io.trino.plugin.hive.TestingHiveUtils.getConnectorService; import static io.trino.plugin.tpch.ColumnNaming.SIMPLIFIED; import static io.trino.plugin.tpch.DecimalTypeMapping.DOUBLE; @@ -100,11 +102,13 @@ public static class Builder> private List> initialTables = ImmutableList.of(); private Optional initialSchemasLocationBase = Optional.empty(); private Optional> metastore = Optional.empty(); + private boolean metastoreImpersonationEnabled; private boolean tpcdsCatalogEnabled; private boolean tpchBucketedCatalogEnabled; private boolean createTpchSchemas = true; private ColumnNaming tpchColumnNaming = SIMPLIFIED; private DecimalTypeMapping tpchDecimalTypeMapping = DOUBLE; + private Optional decryptionKeyRetriever = Optional.empty(); protected Builder() { @@ -159,6 +163,13 @@ public SELF setMetastore(Function metasto return self(); } + @CanIgnoreReturnValue + public SELF setMetastoreImpersonationEnabled(boolean metastoreImpersonationEnabled) + { + this.metastoreImpersonationEnabled = metastoreImpersonationEnabled; + return self(); + } + @CanIgnoreReturnValue public SELF setTpcdsCatalogEnabled(boolean tpcdsCatalogEnabled) { @@ -196,6 +207,13 @@ public SELF setTpchDecimalTypeMapping(DecimalTypeMapping tpchDecimalTypeMapping) return self(); } + @CanIgnoreReturnValue + public SELF setDecryptionKeyRetriever(DecryptionKeyRetriever decryptionKeyRetriever) + { + this.decryptionKeyRetriever = Optional.of(requireNonNull(decryptionKeyRetriever, "decryptionKeyRetriever is null")); + return self(); + } + @Override public DistributedQueryRunner build() throws Exception @@ -227,7 +245,7 @@ public DistributedQueryRunner build() hiveProperties.put("fs.hadoop.enabled", "true"); } - queryRunner.installPlugin(new TestingHivePlugin(dataDir, metastore)); + queryRunner.installPlugin(new TestingHivePlugin(dataDir, metastore, metastoreImpersonationEnabled, decryptionKeyRetriever)); Map hiveProperties = new HashMap<>(); if (!skipTimezoneSetup) { @@ -271,7 +289,7 @@ public DistributedQueryRunner build() private void populateData(QueryRunner queryRunner) { HiveMetastore metastore = getConnectorService(queryRunner, HiveMetastoreFactory.class) - .createMetastore(Optional.empty()); + .createMetastore(Optional.of(SESSION.getIdentity())); if (metastore.getDatabase(TPCH_SCHEMA).isEmpty()) { metastore.createDatabase(createDatabaseMetastoreObject(TPCH_SCHEMA, initialSchemasLocationBase)); copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, initialTables); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java index 85a5a62edde9..3a96d1816d14 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/HiveTestUtils.java @@ -58,6 +58,7 @@ import io.trino.plugin.hive.parquet.ParquetReaderConfig; import io.trino.plugin.hive.parquet.ParquetWriterConfig; import io.trino.plugin.hive.rcfile.RcFilePageSourceFactory; +import io.trino.spi.NodeVersion; import io.trino.spi.PageSorter; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.type.ArrayType; @@ -178,7 +179,7 @@ public static Set getDefaultHivePageSourceFactories(Trino .add(new AvroPageSourceFactory(fileSystemFactory)) .add(new RcFilePageSourceFactory(fileSystemFactory, hiveConfig)) .add(new OrcPageSourceFactory(new OrcReaderConfig(), fileSystemFactory, stats, hiveConfig)) - .add(new ParquetPageSourceFactory(fileSystemFactory, stats, new ParquetReaderConfig(), hiveConfig)) + .add(new ParquetPageSourceFactory(fileSystemFactory, stats, Optional.empty(), new ParquetReaderConfig(), hiveConfig)) .add(new ProtobufSequenceFilePageSourceFactory(fileSystemFactory, hiveConfig)) .build(); } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java index f92f86579369..14f0572711c2 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestBackgroundHiveSplitLoader.java @@ -47,6 +47,7 @@ import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.ConnectorSplitSource.ConnectorSplitBatch; +import io.trino.spi.connector.Constraint; import io.trino.spi.connector.DynamicFilter; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.predicate.Domain; @@ -154,7 +155,7 @@ public void tearDown() public void testNoPathFilter() throws Exception { - BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader(TEST_LOCATIONS, TupleDomain.none()); + BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader(TEST_LOCATIONS, TupleDomain.none(), Constraint.alwaysTrue()); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); @@ -205,13 +206,31 @@ public void testPathFilter() { BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_LOCATIONS, - LOCATION_DOMAIN); + LOCATION_DOMAIN, + Constraint.alwaysTrue()); + + HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); + backgroundHiveSplitLoader.start(hiveSplitSource); + List paths = drain(hiveSplitSource); + assertThat(paths).containsExactly(LOCATION.toString()); + } + + @Test + public void testPathConstraint() + throws Exception + { + BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( + TEST_LOCATIONS, + LOCATION_DOMAIN, + new Constraint( + TupleDomain.all(), + LOCATION_DOMAIN.transformKeys(ColumnHandle.class::cast).asPredicate(), + Set.of(pathColumnHandle()))); HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); List paths = drain(hiveSplitSource); - assertThat(paths).hasSize(1); - assertThat(paths.get(0)).isEqualTo(LOCATION.toString()); + assertThat(paths).containsExactly(LOCATION.toString()); } @Test @@ -221,6 +240,7 @@ public void testPathFilterOneBucketMatchPartitionedTable() BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_LOCATIONS, LOCATION_DOMAIN, + Constraint.alwaysTrue(), Optional.of(new HiveBucketFilter(Set.of(0, 1))), PARTITIONED_TABLE, Optional.of(new HiveTablePartitioning(true, BUCKETING_V1, BUCKET_COUNT, BUCKET_COLUMN_HANDLES, false, List.of(), true))); @@ -228,8 +248,7 @@ public void testPathFilterOneBucketMatchPartitionedTable() HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); List paths = drain(hiveSplitSource); - assertThat(paths).hasSize(1); - assertThat(paths.get(0)).isEqualTo(LOCATION.toString()); + assertThat(paths).containsExactly(LOCATION.toString()); } @Test @@ -239,6 +258,7 @@ public void testPathFilterBucketedPartitionedTable() BackgroundHiveSplitLoader backgroundHiveSplitLoader = backgroundHiveSplitLoader( TEST_LOCATIONS, LOCATION_DOMAIN, + Constraint.alwaysTrue(), Optional.empty(), PARTITIONED_TABLE, Optional.of( @@ -254,8 +274,7 @@ public void testPathFilterBucketedPartitionedTable() HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader); backgroundHiveSplitLoader.start(hiveSplitSource); List paths = drain(hiveSplitSource); - assertThat(paths).hasSize(1); - assertThat(paths.get(0)).isEqualTo(LOCATION.toString()); + assertThat(paths).containsExactly(LOCATION.toString()); } @Test @@ -474,6 +493,7 @@ public HivePartitionMetadata next() } }, TupleDomain.all(), + Constraint.alwaysTrue(), DynamicFilter.EMPTY, new Duration(0, SECONDS), TESTING_TYPE_MANAGER, @@ -796,6 +816,7 @@ public void testBuildManifestFileIterator() schema, List.of(), TupleDomain.all(), + Constraint.alwaysTrue(), () -> true, ImmutableMap.of(), Optional.empty(), @@ -837,6 +858,7 @@ public void testBuildManifestFileIteratorNestedDirectory() schema, List.of(), TupleDomain.all(), + Constraint.alwaysTrue(), () -> true, ImmutableMap.of(), Optional.empty(), @@ -875,6 +897,7 @@ public void testBuildManifestFileIteratorWithCacheInvalidation() schema, List.of(), TupleDomain.all(), + Constraint.alwaysTrue(), () -> true, ImmutableMap.of(), Optional.empty(), @@ -1056,6 +1079,7 @@ private BackgroundHiveSplitLoader backgroundHiveSplitLoader( return backgroundHiveSplitLoader( fileSystemFactory, TupleDomain.all(), + Constraint.alwaysTrue(), dynamicFilter, dynamicFilteringProbeBlockingTimeoutMillis, Optional.empty(), @@ -1079,12 +1103,14 @@ private static TrinoFileSystemFactory createTestingFileSystem(Collection locations, - TupleDomain tupleDomain) + TupleDomain tupleDomain, + Constraint constraint) throws IOException { return backgroundHiveSplitLoader( locations, tupleDomain, + constraint, Optional.empty(), SIMPLE_TABLE, Optional.empty()); @@ -1093,37 +1119,23 @@ private BackgroundHiveSplitLoader backgroundHiveSplitLoader( private BackgroundHiveSplitLoader backgroundHiveSplitLoader( List locations, TupleDomain compactEffectivePredicate, + Constraint constraint, Optional hiveBucketFilter, Table table, Optional tablePartitioning) throws IOException - { - return backgroundHiveSplitLoader( - locations, - compactEffectivePredicate, - hiveBucketFilter, - table, - tablePartitioning, - Optional.empty()); - } - - private BackgroundHiveSplitLoader backgroundHiveSplitLoader( - List locations, - TupleDomain compactEffectivePredicate, - Optional hiveBucketFilter, - Table table, - Optional tablePartitioning, - Optional validWriteIds) - throws IOException { TrinoFileSystemFactory fileSystemFactory = createTestingFileSystem(locations); return backgroundHiveSplitLoader( fileSystemFactory, compactEffectivePredicate, + constraint, + DynamicFilter.EMPTY, + new Duration(0, SECONDS), hiveBucketFilter, table, tablePartitioning, - validWriteIds); + Optional.empty()); } private BackgroundHiveSplitLoader backgroundHiveSplitLoader( @@ -1137,6 +1149,7 @@ private BackgroundHiveSplitLoader backgroundHiveSplitLoader( return backgroundHiveSplitLoader( fileSystemFactory, compactEffectivePredicate, + Constraint.alwaysTrue(), DynamicFilter.EMPTY, new Duration(0, SECONDS), hiveBucketFilter, @@ -1148,6 +1161,7 @@ private BackgroundHiveSplitLoader backgroundHiveSplitLoader( private BackgroundHiveSplitLoader backgroundHiveSplitLoader( TrinoFileSystemFactory fileSystemFactory, TupleDomain compactEffectivePredicate, + Constraint constraint, DynamicFilter dynamicFilter, Duration dynamicFilteringProbeBlockingTimeout, Optional hiveBucketFilter, @@ -1166,6 +1180,7 @@ private BackgroundHiveSplitLoader backgroundHiveSplitLoader( table, hivePartitionMetadatas.iterator(), compactEffectivePredicate, + constraint, dynamicFilter, dynamicFilteringProbeBlockingTimeout, TESTING_TYPE_MANAGER, @@ -1210,6 +1225,7 @@ private BackgroundHiveSplitLoader backgroundHiveSplitLoader( SIMPLE_TABLE, partitions.iterator(), TupleDomain.none(), + Constraint.alwaysTrue(), DynamicFilter.EMPTY, new Duration(0, SECONDS), TESTING_TYPE_MANAGER, @@ -1237,6 +1253,7 @@ private BackgroundHiveSplitLoader backgroundHiveSplitLoaderOfflinePartitions() SIMPLE_TABLE, createPartitionMetadataWithOfflinePartitions(), TupleDomain.all(), + Constraint.alwaysTrue(), DynamicFilter.EMPTY, new Duration(0, SECONDS), TESTING_TYPE_MANAGER, diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java index 255a2c728a6d..8f79bf1acfef 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveConfig.java @@ -20,6 +20,7 @@ import io.airlift.units.Duration; import org.junit.jupiter.api.Test; +import java.io.IOException; import java.nio.file.Path; import java.util.Map; import java.util.TimeZone; @@ -128,6 +129,7 @@ public void testDefaults() @Test public void testExplicitPropertyMappings() + throws IOException { Map properties = ImmutableMap.builder() .put("hive.single-statement-writes", "true") diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java index cc7e10059471..95d95365b225 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHiveFileFormats.java @@ -52,6 +52,7 @@ import io.trino.plugin.hive.parquet.ParquetWriterConfig; import io.trino.plugin.hive.rcfile.RcFilePageSourceFactory; import io.trino.plugin.hive.util.HiveTypeTranslator; +import io.trino.spi.NodeVersion; import io.trino.spi.Page; import io.trino.spi.PageBuilder; import io.trino.spi.block.ArrayBlockBuilder; @@ -554,7 +555,7 @@ public void testParquetPageSource(int rowCount, long fileSizePadding) .withSession(PARQUET_SESSION) .withRowsCount(rowCount) .withFileSizePadding(fileSizePadding) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); } @Test(dataProvider = "validRowAndFileSizePadding") @@ -568,7 +569,7 @@ public void testParquetPageSourceGzip(int rowCount, long fileSizePadding) .withCompressionCodec(HiveCompressionCodec.GZIP) .withFileSizePadding(fileSizePadding) .withRowsCount(rowCount) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -583,7 +584,7 @@ public void testParquetWriter(int rowCount) .withColumns(testColumns) .withRowsCount(rowCount) .withFileWriterFactory(fileSystemFactory -> new ParquetFileWriterFactory(fileSystemFactory, new NodeVersion("test-version"), TESTING_TYPE_MANAGER, new HiveConfig(), STATS)) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -601,7 +602,7 @@ public void testParquetPageSourceSchemaEvolution(int rowCount) .withReadColumns(readColumns) .withSession(PARQUET_SESSION) .withRowsCount(rowCount) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); // test the name-based access readColumns = writeColumns.reversed(); @@ -609,7 +610,7 @@ public void testParquetPageSourceSchemaEvolution(int rowCount) .withWriteColumns(writeColumns) .withReadColumns(readColumns) .withSession(PARQUET_SESSION_USE_NAME) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -627,7 +628,7 @@ public void testParquetCaseSensitivity(int rowCount) .withSession(getHiveSession(createParquetHiveConfig(true), new ParquetWriterConfig().setValidationPercentage(0))) .withRowsCount(rowCount) .withFileWriterFactory(fileSystemFactory -> new ParquetFileWriterFactory(fileSystemFactory, new NodeVersion("test-version"), TESTING_TYPE_MANAGER, new HiveConfig(), STATS)) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); } private static List getTestColumnsSupportedByParquet() @@ -670,7 +671,7 @@ public void testTruncateVarcharColumn() .withWriteColumns(ImmutableList.of(writeColumn)) .withReadColumns(ImmutableList.of(readColumn)) .withSession(PARQUET_SESSION) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); assertThatFileFormat(AVRO) .withWriteColumns(ImmutableList.of(writeColumn)) @@ -736,14 +737,14 @@ public void testParquetProjectedColumns(int rowCount) .withReadColumns(readColumns) .withRowsCount(rowCount) .withSession(PARQUET_SESSION) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); assertThatFileFormat(PARQUET) .withWriteColumns(writeColumns) .withReadColumns(readColumns) .withRowsCount(rowCount) .withSession(PARQUET_SESSION_USE_NAME) - .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig())); + .isReadableByPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig())); } @Test(dataProvider = "rowCount") @@ -944,7 +945,7 @@ public void testFailForLongVarcharPartitionColumn() assertThatFileFormat(PARQUET) .withColumns(columns) .withSession(PARQUET_SESSION) - .isFailingForPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, new ParquetReaderConfig(), new HiveConfig()), expectedErrorCode, expectedMessage); + .isFailingForPageSource(fileSystemFactory -> new ParquetPageSourceFactory(fileSystemFactory, STATS, Optional.empty(), new ParquetReaderConfig(), new HiveConfig()), expectedErrorCode, expectedMessage); } private static void testPageSourceFactory( diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java index bc2c7786a81e..182bb5127f05 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSink.java @@ -394,7 +394,7 @@ private static ConnectorPageSink createPageSink( getDefaultHiveFileWriterFactories(config, fileSystemFactory), HDFS_FILE_SYSTEM_FACTORY, PAGE_SORTER, - HiveMetastoreFactory.ofInstance(metastore), + HiveMetastoreFactory.ofInstance(metastore, false), new GroupByHashPageIndexerFactory(new FlatHashStrategyCompiler(new TypeOperators())), TESTING_TYPE_MANAGER, config, diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSourceProvider.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSourceProvider.java index 6f7886f3a49c..d8cabe9874c9 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSourceProvider.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestHivePageSourceProvider.java @@ -207,8 +207,8 @@ private static Block createProjectedColumnBlock(Block data, Type finalType, RowT } else { int lastDereference = dereferences.getLast(); - - finalType.appendTo(currentData.getRawFieldBlock(lastDereference), currentData.getRawIndex(), builder); + Block fieldBlock = currentData.getRawFieldBlock(lastDereference); + builder.append(fieldBlock.getUnderlyingValueBlock(), fieldBlock.getUnderlyingValuePosition(currentData.getRawIndex())); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestThriftMetastoreImpersonation.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestThriftMetastoreImpersonation.java new file mode 100644 index 000000000000..18595bd7dc07 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestThriftMetastoreImpersonation.java @@ -0,0 +1,73 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive; + +import com.google.common.collect.ImmutableMap; +import io.trino.Session; +import io.trino.plugin.hive.containers.Hive3MinioDataLake; +import io.trino.plugin.hive.containers.HiveMinioDataLake; +import io.trino.plugin.hive.s3.S3HiveQueryRunner; +import io.trino.spi.security.Identity; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import io.trino.testing.sql.TestTable; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.util.List; + +import static io.trino.plugin.hive.containers.HiveHadoop.HIVE3_IMAGE; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +final class TestThriftMetastoreImpersonation + extends AbstractTestQueryFramework +{ + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + HiveMinioDataLake hiveMinioDataLake = closeAfterClass(new Hive3MinioDataLake("test-thrift-impersonation-" + randomNameSuffix(), HIVE3_IMAGE)); + hiveMinioDataLake.start(); + return S3HiveQueryRunner.builder(hiveMinioDataLake) + .setMetastoreImpersonationEnabled(true) + .setHiveProperties(ImmutableMap.builder() + .put("hive.security", "allow-all") + .put("hive.metastore-cache-ttl", "1d") + .put("hive.user-metastore-cache-ttl", "1d") + .buildOrThrow()) + .build(); + } + + @Test + void testFlushMetadataCache() + { + Session alice = Session.builder(getSession()).setIdentity(Identity.ofUser("alice")).build(); + + try (TestTable table = newTrinoTable("test_partition", "(id int, part int) WITH (partitioned_by = ARRAY['part'])", List.of("1, 10"))) { + assertThat(computeScalar(alice, "SELECT count(1) FROM \"" + table.getName() + "$partitions\"")) + .isEqualTo(1L); + + assertUpdate("INSERT INTO " + table.getName() + " VALUES (2, 20)", 1); + assertThat(computeScalar(alice, "SELECT count(1) FROM \"" + table.getName() + "$partitions\"")) + .isEqualTo(1L); + + assertUpdate(alice, "CALL system.flush_metadata_cache(schema_name => CURRENT_SCHEMA, table_name => '" + table.getName() + "')"); + assertThat(computeScalar(alice, "SELECT count(1) FROM \"" + table.getName() + "$partitions\"")) + .isEqualTo(2L); + } + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHiveConnectorFactory.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHiveConnectorFactory.java index c0e03b111e10..825a3435c37e 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHiveConnectorFactory.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHiveConnectorFactory.java @@ -15,9 +15,11 @@ import com.google.common.collect.ImmutableMap; import com.google.inject.Module; +import com.google.inject.multibindings.Multibinder; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.local.LocalFileSystemFactory; import io.trino.metastore.HiveMetastore; +import io.trino.parquet.crypto.DecryptionKeyRetriever; import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -36,23 +38,35 @@ public class TestingHiveConnectorFactory implements ConnectorFactory { private final Optional metastore; + private final boolean metastoreImpersonationEnabled; private final Module module; public TestingHiveConnectorFactory(Path localFileSystemRootPath) { - this(localFileSystemRootPath, Optional.empty()); + this(localFileSystemRootPath, Optional.empty(), false, Optional.empty()); } @Deprecated - public TestingHiveConnectorFactory(Path localFileSystemRootPath, Optional metastore) + public TestingHiveConnectorFactory( + Path localFileSystemRootPath, + Optional metastore, + boolean metastoreImpersonationEnabled, + Optional decryptionKeyRetriever) { this.metastore = requireNonNull(metastore, "metastore is null"); + this.metastoreImpersonationEnabled = metastoreImpersonationEnabled; boolean ignored = localFileSystemRootPath.toFile().mkdirs(); this.module = binder -> { newMapBinder(binder, String.class, TrinoFileSystemFactory.class) .addBinding("local").toInstance(new LocalFileSystemFactory(localFileSystemRootPath)); configBinder(binder).bindConfigDefaults(FileHiveMetastoreConfig.class, config -> config.setCatalogDirectory("local:///")); + + decryptionKeyRetriever.ifPresent(retriever -> { + Multibinder retrieverBinder = + Multibinder.newSetBinder(binder, DecryptionKeyRetriever.class); + retrieverBinder.addBinding().toInstance(retriever); + }); }; } @@ -71,6 +85,6 @@ public Connector create(String catalogName, Map config, Connecto if (metastore.isEmpty() && !config.containsKey("hive.metastore")) { configBuilder.put("hive.metastore", "file"); } - return createConnector(catalogName, configBuilder.buildOrThrow(), context, module, metastore, Optional.empty()); + return createConnector(catalogName, configBuilder.buildOrThrow(), context, module, metastore, metastoreImpersonationEnabled, Optional.empty()); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHivePlugin.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHivePlugin.java index 41f492a99405..a42cf85530f8 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHivePlugin.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingHivePlugin.java @@ -15,6 +15,7 @@ import com.google.common.collect.ImmutableList; import io.trino.metastore.HiveMetastore; +import io.trino.parquet.crypto.DecryptionKeyRetriever; import io.trino.spi.Plugin; import io.trino.spi.connector.ConnectorFactory; @@ -28,28 +29,36 @@ public class TestingHivePlugin { private final Path localFileSystemRootPath; private final Optional metastore; + private final boolean metastoreImpersonationEnabled; + private final Optional decryptionKeyRetriever; public TestingHivePlugin(Path localFileSystemRootPath) { - this(localFileSystemRootPath, Optional.empty()); + this(localFileSystemRootPath, Optional.empty(), false, Optional.empty()); } @Deprecated public TestingHivePlugin(Path localFileSystemRootPath, HiveMetastore metastore) { - this(localFileSystemRootPath, Optional.of(metastore)); + this(localFileSystemRootPath, Optional.of(metastore), false, Optional.empty()); } @Deprecated - public TestingHivePlugin(Path localFileSystemRootPath, Optional metastore) + public TestingHivePlugin( + Path localFileSystemRootPath, + Optional metastore, + boolean metastoreImpersonationEnabled, + Optional decryptionKeyRetriever) { this.localFileSystemRootPath = requireNonNull(localFileSystemRootPath, "localFileSystemRootPath is null"); this.metastore = requireNonNull(metastore, "metastore is null"); + this.metastoreImpersonationEnabled = metastoreImpersonationEnabled; + this.decryptionKeyRetriever = requireNonNull(decryptionKeyRetriever, "decryptionKeyRetriever is null"); } @Override public Iterable getConnectorFactories() { - return ImmutableList.of(new TestingHiveConnectorFactory(localFileSystemRootPath, metastore)); + return ImmutableList.of(new TestingHiveConnectorFactory(localFileSystemRootPath, metastore, metastoreImpersonationEnabled, decryptionKeyRetriever)); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingThriftHiveMetastoreBuilder.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingThriftHiveMetastoreBuilder.java index c344df637391..f1f31e81ad1e 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingThriftHiveMetastoreBuilder.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/TestingThriftHiveMetastoreBuilder.java @@ -35,6 +35,7 @@ import static io.trino.plugin.base.security.UserNameProvider.SIMPLE_USER_NAME_PROVIDER; import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT; import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS; +import static io.trino.plugin.hive.HiveTestUtils.SESSION; import static io.trino.plugin.hive.metastore.thrift.TestingTokenAwareMetastoreClientFactory.TIMEOUT; import static java.util.Objects.requireNonNull; import static java.util.concurrent.Executors.newFixedThreadPool; @@ -108,6 +109,6 @@ public ThriftMetastore build(Consumer registerResource) thriftMetastoreConfig, fileSystemFactory, executorService); - return metastoreFactory.createMetastore(Optional.empty()); + return metastoreFactory.createMetastore(Optional.of(SESSION.getIdentity())); } } diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestEnvironmentDecryptionKeyRetriever.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestEnvironmentDecryptionKeyRetriever.java new file mode 100644 index 000000000000..66537a8e478c --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestEnvironmentDecryptionKeyRetriever.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.crypto; + +import org.apache.parquet.hadoop.metadata.ColumnPath; +import org.testng.annotations.Test; + +import java.util.Base64; +import java.util.Optional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; + +public final class TestEnvironmentDecryptionKeyRetriever +{ + private static final ColumnPath AGE = ColumnPath.fromDotString("age"); + + @Test + public void defaultEmpty() + { + EnvironmentDecryptionKeyRetriever retriever = new EnvironmentDecryptionKeyRetriever(null, null); + + assertThat(retriever.getFooterKey(Optional.empty())).isEmpty(); + assertThat(retriever.getColumnKey(AGE, Optional.empty())).isEmpty(); + } + + @Test + public void singleKeyMode() + { + EnvironmentDecryptionKeyRetriever retriever = new EnvironmentDecryptionKeyRetriever(b64("foot"), b64("colKey")); + + assertThat(retriever.getFooterKey(Optional.of("ignored".getBytes(UTF_8)))).contains("foot".getBytes(UTF_8)); + assertThat(retriever.getColumnKey(AGE, Optional.empty())).contains("colKey".getBytes(UTF_8)); + } + + @Test + public void mapModeUsesMetadata() + { + // footer: id1→k1 , id2→k2 + String footerValue = String.join(",", + "id1:" + b64("k1"), "id2:" + b64("k2")); + // column: meta→ageKey + String columnValue = "meta:" + b64("ageKey"); + + EnvironmentDecryptionKeyRetriever retriever = new EnvironmentDecryptionKeyRetriever(footerValue, columnValue); + + assertThat(retriever.getFooterKey(Optional.of("id2".getBytes(UTF_8)))).contains("k2".getBytes(UTF_8)); + assertThat(retriever.getColumnKey(AGE, Optional.of("meta".getBytes(UTF_8)))).contains("ageKey".getBytes(UTF_8)); + + // unknown metadata → empty + assertThat(retriever.getFooterKey(Optional.of("zzz".getBytes(UTF_8)))).isEmpty(); + assertThat(retriever.getColumnKey(AGE, Optional.empty())).isEmpty(); + } + + private static String b64(String string) + { + return Base64.getEncoder().encodeToString(string.getBytes(UTF_8)); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestHiveParquetEncryption.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestHiveParquetEncryption.java new file mode 100644 index 000000000000..a90e652285eb --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestHiveParquetEncryption.java @@ -0,0 +1,334 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.crypto; + +import com.google.common.collect.ImmutableMap; +import io.trino.filesystem.Location; +import io.trino.parquet.ParquetDataSource; +import io.trino.parquet.ParquetReaderOptions; +import io.trino.parquet.crypto.DecryptionKeyRetriever; +import io.trino.parquet.crypto.FileDecryptionProperties; +import io.trino.parquet.metadata.ColumnChunkMetadata; +import io.trino.parquet.metadata.ParquetMetadata; +import io.trino.parquet.reader.FileParquetDataSource; +import io.trino.parquet.reader.MetadataReader; +import io.trino.plugin.hive.HiveQueryRunner; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.MaterializedResult; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.crypto.ColumnEncryptionProperties; +import org.apache.parquet.crypto.FileEncryptionProperties; +import org.apache.parquet.crypto.ParquetCipher; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.ColumnPath; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +import java.nio.file.Files; +import java.util.Map; +import java.util.Optional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; +import static org.apache.parquet.hadoop.ParquetFileWriter.Mode.OVERWRITE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +/** + * End‑to‑end PME flow: Parquet writer → Hive connector → Trino query. + */ +// ExampleParquetWriter is not thread-safe +@TestInstance(PER_CLASS) +@Execution(ExecutionMode.SAME_THREAD) +public class TestHiveParquetEncryption + extends AbstractTestQueryFramework +{ + private static final byte[] FOOTER_KEY = "footKeyIs16Byte?".getBytes(UTF_8); + + // Keys per column (different on purpose) + private static final byte[] COLUMN_KEY_AGE = "colKeyIs16ByteA?".getBytes(UTF_8); + private static final byte[] COLUMN_KEY_ID = "colKeyIs16ByteB?".getBytes(UTF_8); + + private static final ColumnPath AGE_PATH = ColumnPath.fromDotString("age"); + private static final ColumnPath ID_PATH = ColumnPath.fromDotString("id"); + + /** + * kept so we can reference the warehouse path later + */ + private java.nio.file.Path warehouseDir; + + @Override + protected DistributedQueryRunner createQueryRunner() + throws Exception + { + warehouseDir = Files.createTempDirectory("pme_hive"); + + Map properties = ImmutableMap.builder() + .put("hive.metastore", "file") + .put("hive.metastore.catalog.dir", warehouseDir.toUri().toString()) + .put("pme.environment-key-retriever.enabled", "false") + .buildOrThrow(); + + // Bind retriever that knows ONLY the footer key + age column key + return HiveQueryRunner.builder() + .setHiveProperties(properties) + .setDecryptionKeyRetriever(new TestingParquetEncryptionModule(FOOTER_KEY, Optional.of(COLUMN_KEY_AGE), Optional.empty())) + .build(); + } + + @Test + public void testEncryptedParquetRead() + throws Exception + { + // 1) write encrypted file inside warehouse + java.nio.file.Path dataDir = Files.createDirectories(warehouseDir.resolve("pme_data")); + java.nio.file.Path parquetFile = dataDir.resolve("data.parquet"); + writeEncryptedFile(parquetFile); + + // 2) create external table + String location = Location.of(String.valueOf(dataDir.toUri())).toString(); + assertUpdate(""" + CREATE TABLE enc_age(age INT) + WITH (external_location = '%s', format = 'PARQUET') + """.formatted(location)); + + // 3) verify results + MaterializedResult result = computeActual("SELECT COUNT(*), MIN(age), MAX(age) FROM enc_age"); + assertThat(result.getMaterializedRows().get(0).getField(0)).isEqualTo(100L); // count + assertThat(result.getMaterializedRows().get(0).getField(1)).isEqualTo(0); // min + assertThat(result.getMaterializedRows().get(0).getField(2)).isEqualTo(99); // max + } + + @Test + public void testTwoColumnFileOnlyAgeKeyProvided() + throws Exception + { + // 1) write a two-column file (both columns encrypted with different keys) + java.nio.file.Path dataDir = Files.createDirectories(warehouseDir.resolve("pme_two_cols")); + java.nio.file.Path parquetFile = dataDir.resolve("data.parquet"); + writeTwoColumnEncryptedFile(parquetFile); + + // 2) create external table with both columns + String location = Location.of(String.valueOf(dataDir.toUri())).toString(); + assertUpdate(""" + CREATE TABLE enc_two(id INT, age INT) + WITH (external_location = '%s', format = 'PARQUET') + """.formatted(location)); + + // 3) Selecting ONLY the accessible column (age) must succeed + MaterializedResult ok = computeActual("SELECT COUNT(*), MIN(age), MAX(age) FROM enc_two"); + assertThat(ok.getMaterializedRows().get(0).getField(0)).isEqualTo(100L); + assertThat(ok.getMaterializedRows().get(0).getField(1)).isEqualTo(0); + assertThat(ok.getMaterializedRows().get(0).getField(2)).isEqualTo(99); + + // 4) Selecting the inaccessible column (id) must fail (no column key) + // We match on a broad error text to avoid coupling to a specific message. + assertQueryFails("SELECT MIN(id) FROM enc_two", "(?s).*User does not have access to column.*"); + } + + @Test + public void testEncryptedDictionaryPruningTwoColumns() + throws Exception + { + // 1) write an encrypted, dictionary-encoded two-column file + // Values are 0..10 except the “missing” ones; RG min/max is [0,10] so min/max alone can’t prune. + java.nio.file.Path dataDir = Files.createDirectories(warehouseDir.resolve("pme_dict2_enc")); + java.nio.file.Path parquetFile = dataDir.resolve("data.parquet"); + int missingAge = 7; + int missingId = 3; + writeTwoColumnEncryptedDictionaryFile(parquetFile, missingAge, missingId); + + // 2) create external table + String location = Location.of(String.valueOf(dataDir.toUri())).toString(); + assertUpdate(""" + CREATE TABLE enc_dict2(id INT, age INT) + WITH (external_location = '%s', format = 'PARQUET') + """.formatted(location)); + + // 3) Predicate on the accessible column (age) – dictionary (encrypted) gets read & prunes to zero + assertThat(computeActual("SELECT count(*) FROM enc_dict2 WHERE age = " + missingAge).getOnlyValue()) + .isEqualTo(0L); + + // sanity: present value on age returns rows + assertThat(((Number) computeActual("SELECT count(*) FROM enc_dict2 WHERE age = 5").getOnlyValue()).longValue()) + .isGreaterThan(0L); + + // 4) Predicate on the inaccessible column (id) – should fail (no column key) + assertQueryFails("SELECT count(*) FROM enc_dict2 WHERE id = " + missingId, "(?s).*access.*column.*id.*"); + + assertQuerySucceeds("DROP TABLE enc_dict2"); + } + + // ───────────────────────── writer ───────────────────────── + private static void writeEncryptedFile(java.nio.file.Path path) + throws Exception + { + var schema = MessageTypeParser.parseMessageType( + "message doc { required int32 age; }"); + + // This test purposely reuses one demo key. + // With Parquet Modular Encryption (AES-GCM/CTR), reusing a key for lots of data + // weakens security. + ColumnEncryptionProperties columnProperties = ColumnEncryptionProperties.builder(AGE_PATH) + .withKey(COLUMN_KEY_AGE) + .build(); + + FileEncryptionProperties encodingProperties = FileEncryptionProperties.builder(FOOTER_KEY) + .withAlgorithm(ParquetCipher.AES_GCM_CTR_V1) + .withEncryptedColumns(Map.of(AGE_PATH, columnProperties)) + .build(); + + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toString())) + .withType(schema) + .withConf(new Configuration()) + .withEncryption(encodingProperties) + .withWriteMode(OVERWRITE) + .build()) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < 100; i++) { + writer.write(factory.newGroup().append("age", i)); + } + } + } + + private static void writeTwoColumnEncryptedFile(java.nio.file.Path path) + throws Exception + { + var schema = MessageTypeParser.parseMessageType("message doc { required int32 age; required int32 id; }"); + + ColumnEncryptionProperties idProperties = ColumnEncryptionProperties.builder(ID_PATH).withKey(COLUMN_KEY_ID).build(); + ColumnEncryptionProperties ageProperties = ColumnEncryptionProperties.builder(AGE_PATH).withKey(COLUMN_KEY_AGE).build(); + + FileEncryptionProperties encodingProperties = FileEncryptionProperties.builder(FOOTER_KEY) + .withAlgorithm(ParquetCipher.AES_GCM_CTR_V1) + .withEncryptedColumns(Map.of(AGE_PATH, ageProperties, ID_PATH, idProperties)) + .build(); + + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toString())) + .withType(schema) + .withConf(new Configuration()) + .withEncryption(encodingProperties) + .withWriteMode(OVERWRITE) + .build()) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < 100; i++) { + writer.write(factory.newGroup().append("id", 100 - i).append("age", i)); + } + } + } + + /** + * Two-column writer: + * - both columns encrypted (different keys), + * - tiny page size to encourage dictionary encoding, + * - each column skips one value in 0..10 so dictionary-based pruning can eliminate the row-group. + * Reader in this test only has the AGE key (ID key is absent). + */ + private static void writeTwoColumnEncryptedDictionaryFile(java.nio.file.Path path, int missingAge, int missingId) + throws Exception + { + var schema = MessageTypeParser.parseMessageType("message doc { required int32 age; required int32 id; }"); + + ColumnEncryptionProperties idProperties = ColumnEncryptionProperties.builder(ID_PATH).withKey(COLUMN_KEY_ID).build(); + ColumnEncryptionProperties ageProperties = ColumnEncryptionProperties.builder(AGE_PATH).withKey(COLUMN_KEY_AGE).build(); + + FileEncryptionProperties encodingProperties = FileEncryptionProperties.builder(FOOTER_KEY) + .withAlgorithm(ParquetCipher.AES_GCM_CTR_V1) + .withEncryptedColumns(Map.of(AGE_PATH, ageProperties, ID_PATH, idProperties)) + .build(); + + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toString())) + .withType(schema) + .withConf(new Configuration()) + .withEncryption(encodingProperties) + .withWriteMode(OVERWRITE) + .withPageSize(1024) // small pages -> dictionary likely + .build()) { + SimpleGroupFactory factory = new SimpleGroupFactory(schema); + for (int i = 0; i < 5000; i++) { + int id = i % 11; + if (id == missingId) { + id = (id + 1) % 11; // skip one value for id + } + int age = i % 11; + if (age == missingAge) { + age = (age + 1) % 11; // skip one value for age + } + writer.write(factory.newGroup().append("id", id).append("age", age)); + } + } + + // Verify both columns actually have dictionary pages + try (ParquetDataSource source = new FileParquetDataSource(path.toFile(), ParquetReaderOptions.defaultOptions())) { + FileDecryptionProperties dec = FileDecryptionProperties.builder() + .withKeyRetriever(new TestHiveParquetEncryption.TestingParquetEncryptionModule( + FOOTER_KEY, Optional.of(COLUMN_KEY_AGE), Optional.of(COLUMN_KEY_ID))) + .build(); + ParquetMetadata metadata = MetadataReader.readFooter(source, Optional.empty(), Optional.empty(), Optional.of(dec)); + + ColumnChunkMetadata ageChunk = metadata.getBlocks().getFirst().columns().stream() + .filter(column -> column.getPath().equals(AGE_PATH)) + .findFirst().orElseThrow(); + ColumnChunkMetadata idChunk = metadata.getBlocks().getFirst().columns().stream() + .filter(column -> column.getPath().equals(ID_PATH)) + .findFirst().orElseThrow(); + + assertThat(ageChunk.getDictionaryPageOffset()).isGreaterThan(0); + assertThat(idChunk.getDictionaryPageOffset()).isGreaterThan(0); + assertThat(ageChunk.getEncodings()).anyMatch(org.apache.parquet.column.Encoding::usesDictionary); + assertThat(idChunk.getEncodings()).anyMatch(org.apache.parquet.column.Encoding::usesDictionary); + } + } + + public static class TestingParquetEncryptionModule + implements DecryptionKeyRetriever + { + private final byte[] footerKey; + private final Optional ageKey; + private final Optional idKey; + + public TestingParquetEncryptionModule(byte[] footerKey, Optional ageKey, Optional idKey) + { + this.footerKey = requireNonNull(footerKey, "footerKey is null"); + this.ageKey = requireNonNull(ageKey, "ageKey is null"); + this.idKey = requireNonNull(idKey, "idKey is null"); + } + + @Override + public Optional getColumnKey(ColumnPath columnPath, Optional keyMetadata) + { + String path = columnPath.toDotString(); + if ("age".equals(path)) { + return ageKey; + } + if ("id".equals(path)) { + return idKey; + } + return Optional.empty(); + } + + @Override + public Optional getFooterKey(Optional keyMetadata) + { + return Optional.of(footerKey); + } + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestParquetEncryptionConfig.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestParquetEncryptionConfig.java new file mode 100644 index 000000000000..641863e261d7 --- /dev/null +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/crypto/TestParquetEncryptionConfig.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hive.crypto; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestParquetEncryptionConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(ParquetEncryptionConfig.class) + .setEnvironmentKeyRetrieverEnabled(false) + .setAadPrefix(null) + .setCheckFooterIntegrity(true)); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("pme.environment-key-retriever.enabled", "true") + .put("pme.aad-prefix", "tenant‑42") + .put("pme.check-footer-integrity", "false") + .buildOrThrow(); + + ParquetEncryptionConfig expected = new ParquetEncryptionConfig() + .setEnvironmentKeyRetrieverEnabled(true) + .setAadPrefix("tenant‑42") + .setCheckFooterIntegrity(false); + + assertFullMapping(properties, expected); + } +} diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestFileHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestFileHiveMetastore.java index 281f14c730fb..6373e74a8752 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestFileHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestFileHiveMetastore.java @@ -20,8 +20,8 @@ import io.trino.metastore.HiveMetastore; import io.trino.metastore.StorageFormat; import io.trino.metastore.Table; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.AbstractTestHiveMetastore; +import io.trino.spi.NodeVersion; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestingFileHiveMetastore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestingFileHiveMetastore.java index 360e613cfbae..cb87d01ff4a9 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestingFileHiveMetastore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/metastore/file/TestingFileHiveMetastore.java @@ -16,8 +16,8 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.local.LocalFileSystemFactory; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.HiveMetastoreConfig; +import io.trino.spi.NodeVersion; import java.io.File; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestOrcPredicates.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestOrcPredicates.java index e7f61efbfa01..b9e1df49ae35 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestOrcPredicates.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/orc/TestOrcPredicates.java @@ -30,10 +30,10 @@ import io.trino.plugin.hive.HiveCompressionCodec; import io.trino.plugin.hive.HiveConfig; import io.trino.plugin.hive.HivePageSourceProvider; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.Schema; import io.trino.plugin.hive.WriterKind; import io.trino.plugin.hive.util.HiveTypeTranslator; +import io.trino.spi.NodeVersion; import io.trino.spi.Page; import io.trino.spi.block.Block; import io.trino.spi.block.IntArrayBlock; diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/ParquetUtil.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/ParquetUtil.java index d93a46589c29..e4ad205c98ac 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/ParquetUtil.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/ParquetUtil.java @@ -80,6 +80,7 @@ private static ConnectorPageSource createPageSource(ConnectorSession session, Fi HivePageSourceFactory hivePageSourceFactory = new ParquetPageSourceFactory( fileSystemFactory, new FileFormatDataSourceStats(), + Optional.empty(), new ParquetReaderConfig(), hiveConfig); diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestBloomFilterStore.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestBloomFilterStore.java index 0f5795aa03c5..fe049c0acfd0 100644 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestBloomFilterStore.java +++ b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/parquet/TestBloomFilterStore.java @@ -308,10 +308,10 @@ private static BloomFilterStore generateBloomFilterStore(ParquetTester.TempFile TrinoInputFile inputFile = new LocalInputFile(tempFile.getFile()); TrinoParquetDataSource dataSource = new TrinoParquetDataSource(inputFile, ParquetReaderOptions.defaultOptions(), new FileFormatDataSourceStats()); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, Optional.empty()); ColumnChunkMetadata columnChunkMetaData = getOnlyElement(getOnlyElement(parquetMetadata.getBlocks()).columns()); - return new BloomFilterStore(dataSource, getOnlyElement(parquetMetadata.getBlocks()), Set.of(columnChunkMetaData.getPath())); + return new BloomFilterStore(dataSource, getOnlyElement(parquetMetadata.getBlocks()), Set.of(columnChunkMetaData.getPath()), Optional.empty()); } private static class BloomFilterTypeTestCase diff --git a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveBlockEncodingSerde.java b/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveBlockEncodingSerde.java deleted file mode 100644 index 924139de39cf..000000000000 --- a/plugin/trino-hive/src/test/java/io/trino/plugin/hive/util/TestHiveBlockEncodingSerde.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.plugin.hive.util; - -import io.airlift.slice.BasicSliceInput; -import io.airlift.slice.DynamicSliceOutput; -import io.airlift.slice.Slice; -import io.airlift.slice.SliceOutput; -import io.trino.spi.block.Block; -import io.trino.spi.type.Type; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.io.UncheckedIOException; - -import static io.trino.spi.type.BigintType.BIGINT; -import static io.trino.spi.type.BooleanType.BOOLEAN; -import static io.trino.spi.type.CharType.createCharType; -import static io.trino.spi.type.DateType.DATE; -import static io.trino.spi.type.DecimalType.createDecimalType; -import static io.trino.spi.type.DoubleType.DOUBLE; -import static io.trino.spi.type.IntegerType.INTEGER; -import static io.trino.spi.type.RealType.REAL; -import static io.trino.spi.type.SmallintType.SMALLINT; -import static io.trino.spi.type.TimeType.createTimeType; -import static io.trino.spi.type.TimeWithTimeZoneType.createTimeWithTimeZoneType; -import static io.trino.spi.type.TimestampType.createTimestampType; -import static io.trino.spi.type.TimestampWithTimeZoneType.createTimestampWithTimeZoneType; -import static io.trino.spi.type.TinyintType.TINYINT; -import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; -import static io.trino.spi.type.VarcharType.createVarcharType; -import static org.assertj.core.api.Assertions.assertThat; - -public class TestHiveBlockEncodingSerde -{ - @Test - public void testSerialization() - { - testSerialization(BOOLEAN); - testSerialization(TINYINT); - testSerialization(SMALLINT); - testSerialization(INTEGER); - testSerialization(BIGINT); - testSerialization(REAL); - testSerialization(DOUBLE); - testSerialization(createDecimalType(7, 2)); - testSerialization(createDecimalType(37, 2)); - testSerialization(createCharType(10)); - testSerialization(createVarcharType(10)); - testSerialization(createUnboundedVarcharType()); - testSerialization(DATE); - testSerialization(createTimeType(0)); - testSerialization(createTimeType(3)); - testSerialization(createTimeType(12)); - testSerialization(createTimeWithTimeZoneType(0)); - testSerialization(createTimeWithTimeZoneType(3)); - testSerialization(createTimeWithTimeZoneType(12)); - testSerialization(createTimestampType(0)); - testSerialization(createTimestampType(3)); - testSerialization(createTimestampType(12)); - testSerialization(createTimestampWithTimeZoneType(0)); - testSerialization(createTimestampWithTimeZoneType(3)); - testSerialization(createTimestampWithTimeZoneType(12)); - } - - private void testSerialization(Type type) - { - Slice serialized; - try (SliceOutput sliceOutput = new DynamicSliceOutput(0)) { - Block block = type.createBlockBuilder(null, 0).build(); - new HiveBlockEncodingSerde().writeBlock(sliceOutput, block); - serialized = sliceOutput.slice(); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - try (BasicSliceInput input = serialized.getInput()) { - Block read = new HiveBlockEncodingSerde().readBlock(input); - assertThat(read.getPositionCount()).isEqualTo(0); - } - } -} diff --git a/plugin/trino-http-event-listener/pom.xml b/plugin/trino-http-event-listener/pom.xml index d699ad75a52d..9bed15db5321 100644 --- a/plugin/trino-http-event-listener/pom.xml +++ b/plugin/trino-http-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-http-event-listener/src/main/java/io/trino/plugin/httpquery/HttpEventListenerFactory.java b/plugin/trino-http-event-listener/src/main/java/io/trino/plugin/httpquery/HttpEventListenerFactory.java index 634ed116aedd..e592dec95fbb 100644 --- a/plugin/trino-http-event-listener/src/main/java/io/trino/plugin/httpquery/HttpEventListenerFactory.java +++ b/plugin/trino-http-event-listener/src/main/java/io/trino/plugin/httpquery/HttpEventListenerFactory.java @@ -56,6 +56,7 @@ public EventListener create(Map config, EventListenerContext con Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpEventListener.java b/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpEventListener.java index 305f5928ab42..16d09edf1b5a 100644 --- a/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpEventListener.java +++ b/plugin/trino-http-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpEventListener.java @@ -153,6 +153,7 @@ final class TestHttpEventListener Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), 0L, 0L, 0L, @@ -179,6 +180,7 @@ final class TestHttpEventListener Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + ImmutableMap.of(), Optional.empty()); queryCreatedEvent = new QueryCreatedEvent( diff --git a/plugin/trino-http-server-event-listener/pom.xml b/plugin/trino-http-server-event-listener/pom.xml index 4e1f3fcd3319..d1252c58d119 100644 --- a/plugin/trino-http-server-event-listener/pom.xml +++ b/plugin/trino-http-server-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-http-server-event-listener/src/main/java/io/trino/plugin/httpquery/HttpServerEventListenerFactory.java b/plugin/trino-http-server-event-listener/src/main/java/io/trino/plugin/httpquery/HttpServerEventListenerFactory.java index 8703b716d132..2e841dbc5973 100644 --- a/plugin/trino-http-server-event-listener/src/main/java/io/trino/plugin/httpquery/HttpServerEventListenerFactory.java +++ b/plugin/trino-http-server-event-listener/src/main/java/io/trino/plugin/httpquery/HttpServerEventListenerFactory.java @@ -69,6 +69,7 @@ HttpServerEventListener createInternal(Map config, EventListener Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-http-server-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpServerEventListener.java b/plugin/trino-http-server-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpServerEventListener.java index cf06bea08abc..3d3d22abbb98 100644 --- a/plugin/trino-http-server-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpServerEventListener.java +++ b/plugin/trino-http-server-event-listener/src/test/java/io/trino/plugin/httpquery/TestHttpServerEventListener.java @@ -141,6 +141,7 @@ final class TestHttpServerEventListener Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), 0L, 0L, 0L, @@ -167,6 +168,7 @@ final class TestHttpServerEventListener Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + ImmutableMap.of(), Optional.empty()); queryCompleteEvent = new QueryCompletedEvent( diff --git a/plugin/trino-hudi/pom.xml b/plugin/trino-hudi/pom.xml index 04ed9479cc95..e1c77f151204 100644 --- a/plugin/trino-hudi/pom.xml +++ b/plugin/trino-hudi/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -403,6 +403,20 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + org.basepom.maven duplicate-finder-maven-plugin diff --git a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java index 1e8201c75925..bf3b140dfe11 100644 --- a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java +++ b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiConnectorFactory.java @@ -20,19 +20,14 @@ import io.airlift.bootstrap.Bootstrap; import io.airlift.bootstrap.LifeCycleManager; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.manager.FileSystemModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorPageSourceProvider; import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; import io.trino.plugin.base.classloader.ClassLoaderSafeNodePartitioningProvider; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.plugin.base.session.SessionPropertiesProvider; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.HiveMetastoreModule; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -40,7 +35,6 @@ import io.trino.spi.connector.ConnectorNodePartitioningProvider; import io.trino.spi.connector.ConnectorPageSourceProvider; import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.type.TypeManager; import org.weakref.jmx.guice.MBeanModule; import java.util.Map; @@ -79,22 +73,15 @@ public static Connector createConnector( new MBeanModule(), new JsonModule(), new HudiModule(), - new HiveMetastoreModule(Optional.empty()), - new FileSystemModule(catalogName, context.getCurrentNode().isCoordinator(), context.getOpenTelemetry(), false), + new HiveMetastoreModule(Optional.empty(), false), + new FileSystemModule(catalogName, context, false), new MBeanServerModule(), module.orElse(EMPTY_MODULE), - binder -> { - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getCurrentNode().getVersion())); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }); + new ConnectorContextModule(catalogName, context)); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java index 532b0568fdd2..8851add79312 100644 --- a/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java +++ b/plugin/trino-hudi/src/main/java/io/trino/plugin/hudi/HudiPageSourceProvider.java @@ -230,7 +230,7 @@ private static ConnectorPageSource createPageSource( try { AggregatedMemoryContext memoryContext = newSimpleAggregatedMemoryContext(); dataSource = createDataSource(inputFile, OptionalLong.of(hudiSplit.fileSize()), options, memoryContext, dataSourceStats); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize(), Optional.empty()); FileMetadata fileMetaData = parquetMetadata.getFileMetaData(); MessageType fileSchema = fileMetaData.getSchema(); @@ -271,7 +271,8 @@ private static ConnectorPageSource createPageSource( options, exception -> handleException(dataSourceId, exception), Optional.of(parquetPredicate), - Optional.empty()); + Optional.empty(), + parquetMetadata.getDecryptionContext()); return createParquetPageSource(columns, fileSchema, messageColumn, useColumnNames, parquetReaderProvider); } catch (IOException | RuntimeException e) { diff --git a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java index 84a4cbd60440..0aa6e0eaa733 100644 --- a/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java +++ b/plugin/trino-hudi/src/test/java/io/trino/plugin/hudi/TestHudiConnectorTest.java @@ -49,6 +49,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_DELETE, SUPPORTS_DEREFERENCE_PUSHDOWN, SUPPORTS_INSERT, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MERGE, SUPPORTS_RENAME_COLUMN, SUPPORTS_RENAME_TABLE, diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index 38f5db7e3616..6c13eaef0a05 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -16,7 +16,7 @@ - 0.105.3 + 0.105.6 @@ -670,13 +670,13 @@ org.testcontainers - postgresql + testcontainers test org.testcontainers - testcontainers + testcontainers-postgresql test @@ -698,6 +698,20 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + org.apache.maven.plugins maven-dependency-plugin diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java index 926059a7c6c5..ba95f5f56255 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConfig.java @@ -22,8 +22,10 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.airlift.units.ThreadCount; +import io.trino.filesystem.Location; import io.trino.plugin.hive.HiveCompressionOption; import jakarta.validation.constraints.AssertFalse; +import jakarta.validation.constraints.AssertTrue; import jakarta.validation.constraints.DecimalMax; import jakarta.validation.constraints.DecimalMin; import jakarta.validation.constraints.Max; @@ -86,6 +88,7 @@ public class IcebergConfig private boolean hideMaterializedViewStorageTable = true; private Optional materializedViewsStorageSchema = Optional.empty(); private boolean sortedWritingEnabled = true; + private Optional sortedWritingLocalStagingPath = Optional.empty(); private boolean queryPartitionFilterRequired; private Set queryPartitionFilterRequiredSchemas = ImmutableSet.of(); private int splitManagerThreads = Math.min(Runtime.getRuntime().availableProcessors() * 2, 32); @@ -453,6 +456,32 @@ public IcebergConfig setSortedWritingEnabled(boolean sortedWritingEnabled) return this; } + @NotNull + public Optional getSortedWritingLocalStagingPath() + { + return sortedWritingLocalStagingPath; + } + + @Config("iceberg.sorted-writing.local-staging-path") + @ConfigDescription("Use provided local directory for staging writes to sorted tables. Use ${USER} placeholder to use different location for each user") + public IcebergConfig setSortedWritingLocalStagingPath(String sortedWritingLocalStagingPath) + { + this.sortedWritingLocalStagingPath = Optional.ofNullable(sortedWritingLocalStagingPath); + return this; + } + + @AssertTrue(message = "iceberg.sorted-writing.local-staging-path must not use any prefix other than file:// or local://") + public boolean isSortedWritingLocalStagingPathValid() + { + if (sortedWritingLocalStagingPath.isEmpty()) { + return true; + } + Optional scheme = Location.of(sortedWritingLocalStagingPath.get()).scheme(); + return scheme.isEmpty() + || scheme.equals(Optional.of("file")) + || scheme.equals(Optional.of("local")); + } + @Config("iceberg.query-partition-filter-required") @ConfigDescription("Require a filter on at least one partition column") public IcebergConfig setQueryPartitionFilterRequired(boolean queryPartitionFilterRequired) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java index e8854cd9c3df..a4ea9e39de38 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergConnectorFactory.java @@ -20,24 +20,16 @@ import io.airlift.bootstrap.Bootstrap; import io.airlift.configuration.AbstractConfigurationAwareModule; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; import io.trino.filesystem.manager.FileSystemModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.plugin.hive.HiveConfig; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.iceberg.catalog.IcebergCatalogModule; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.PageIndexerFactory; -import io.trino.spi.PageSorter; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; import org.weakref.jmx.guice.MBeanModule; import java.util.Map; @@ -82,23 +74,16 @@ public static Connector createConnector( new IcebergSecurityModule(), icebergCatalogModule.orElse(new IcebergCatalogModule()), new MBeanServerModule(), - new IcebergFileSystemModule(catalogName, context.getCurrentNode().isCoordinator(), context.getOpenTelemetry()), + new IcebergFileSystemModule(catalogName, context), + new ConnectorContextModule(catalogName, context), binder -> { binder.bind(ClassLoader.class).toInstance(IcebergConnectorFactory.class.getClassLoader()); - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getCurrentNode().getVersion())); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - binder.bind(PageSorter.class).toInstance(context.getPageSorter()); }, module); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); @@ -112,21 +97,19 @@ private static class IcebergFileSystemModule extends AbstractConfigurationAwareModule { private final String catalogName; - private final boolean isCoordinator; - private final OpenTelemetry openTelemetry; + private final ConnectorContext context; - public IcebergFileSystemModule(String catalogName, boolean isCoordinator, OpenTelemetry openTelemetry) + public IcebergFileSystemModule(String catalogName, ConnectorContext context) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); - this.isCoordinator = isCoordinator; - this.openTelemetry = requireNonNull(openTelemetry, "openTelemetry is null"); + this.context = requireNonNull(context, "context is null"); } @Override protected void setup(Binder binder) { boolean metadataCacheEnabled = buildConfigObject(IcebergConfig.class).isMetadataCacheEnabled(); - install(new FileSystemModule(catalogName, isCoordinator, openTelemetry, metadataCacheEnabled)); + install(new FileSystemModule(catalogName, context, metadataCacheEnabled)); } } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergEnvironmentContext.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergEnvironmentContext.java index 972695368987..b3ebc895c331 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergEnvironmentContext.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergEnvironmentContext.java @@ -14,7 +14,7 @@ package io.trino.plugin.iceberg; import com.google.inject.Inject; -import io.trino.plugin.hive.NodeVersion; +import io.trino.spi.NodeVersion; import org.apache.iceberg.EnvironmentContext; import static java.util.Objects.requireNonNull; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java index 23954907cd95..332b3117ecdc 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergFileWriterFactory.java @@ -32,9 +32,9 @@ import io.trino.plugin.base.metrics.FileFormatDataSourceStats; import io.trino.plugin.hive.HiveCompressionCodec; import io.trino.plugin.hive.HiveCompressionOption; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.orc.OrcWriterConfig; import io.trino.plugin.iceberg.fileio.ForwardingOutputFile; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.type.Type; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java index 65d452344610..390bc22ffe10 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java @@ -1700,8 +1700,8 @@ private Optional getTableHandleForAddFilesFromTable Table icebergTable = catalog.loadTable(session, tableHandle.getSchemaTableName()); checkProcedureArgument( - icebergTable.schemas().size() >= sourceTable.getDataColumns().size(), - "Target table should have at least %d columns but got %d", sourceTable.getDataColumns().size(), icebergTable.schemas().size()); + icebergTable.schema().columns().size() >= sourceTable.getDataColumns().size(), + "Target table should have at least %d columns but got %d", sourceTable.getDataColumns().size(), icebergTable.schema().columns().size()); checkProcedureArgument( icebergTable.spec().fields().size() == sourceTable.getPartitionColumns().size(), "Numbers of partition columns should be equivalent. target: %d, source: %d", icebergTable.spec().fields().size(), sourceTable.getPartitionColumns().size()); @@ -1972,31 +1972,30 @@ private static void commitTransaction(Transaction transaction, String operation) } @Override - public void executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) + public Map executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) { IcebergTableExecuteHandle executeHandle = (IcebergTableExecuteHandle) tableExecuteHandle; switch (executeHandle.procedureId()) { case OPTIMIZE_MANIFESTS: executeOptimizeManifests(session, executeHandle); - return; + return ImmutableMap.of(); case DROP_EXTENDED_STATS: executeDropExtendedStats(session, executeHandle); - return; + return ImmutableMap.of(); case ROLLBACK_TO_SNAPSHOT: executeRollbackToSnapshot(session, executeHandle); - return; + return ImmutableMap.of(); case EXPIRE_SNAPSHOTS: executeExpireSnapshots(session, executeHandle); - return; + return ImmutableMap.of(); case REMOVE_ORPHAN_FILES: - executeRemoveOrphanFiles(session, executeHandle); - return; + return executeRemoveOrphanFiles(session, executeHandle); case ADD_FILES: executeAddFiles(session, executeHandle); - return; + return ImmutableMap.of(); case ADD_FILES_FROM_TABLE: executeAddFilesFromTable(session, executeHandle); - return; + return ImmutableMap.of(); default: throw new IllegalArgumentException("Unknown procedure '" + executeHandle.procedureId() + "'"); } @@ -2118,7 +2117,7 @@ private static void validateTableExecuteParameters( sessionMinRetentionParameterName); } - public void executeRemoveOrphanFiles(ConnectorSession session, IcebergTableExecuteHandle executeHandle) + public Map executeRemoveOrphanFiles(ConnectorSession session, IcebergTableExecuteHandle executeHandle) { IcebergRemoveOrphanFilesHandle removeOrphanFilesHandle = (IcebergRemoveOrphanFilesHandle) executeHandle.procedureHandle(); @@ -2135,14 +2134,14 @@ public void executeRemoveOrphanFiles(ConnectorSession session, IcebergTableExecu if (table.currentSnapshot() == null) { log.debug("Skipping remove_orphan_files procedure for empty table %s", table); - return; + return ImmutableMap.of(); } Instant expiration = session.getStart().minusMillis(retention.toMillis()); - removeOrphanFiles(table, session, executeHandle.schemaTableName(), expiration, executeHandle.fileIoProperties()); + return removeOrphanFiles(table, session, executeHandle.schemaTableName(), expiration, executeHandle.fileIoProperties()); } - private void removeOrphanFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, Instant expiration, Map fileIoProperties) + private Map removeOrphanFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, Instant expiration, Map fileIoProperties) { Set processedManifestFilePaths = new HashSet<>(); // Similarly to issues like https://github.com/trinodb/trino/issues/13759, equivalent paths may have different String @@ -2205,7 +2204,18 @@ private void removeOrphanFiles(Table table, ConnectorSession session, SchemaTabl // Ensure any futures still running are canceled in case of failure manifestScanFutures.forEach(future -> future.cancel(true)); } - scanAndDeleteInvalidFiles(table, session, schemaTableName, expiration, validFileNames, fileIoProperties); + ScanAndDeleteResult result = scanAndDeleteInvalidFiles(table, session, schemaTableName, expiration, validFileNames, fileIoProperties); + log.info("remove_orphan_files for table %s processed %d manifest files, found %d active files, scanned %d files, deleted %d files", + schemaTableName, + processedManifestFilePaths.size(), + validFileNames.size() - 1, // excluding version-hint.text + result.scannedFilesCount(), + result.deletedFilesCount()); + return ImmutableMap.of( + "processed_manifests_count", (long) processedManifestFilePaths.size(), + "active_files_count", (long) validFileNames.size() - 1, // excluding version-hint.text + "scanned_files_count", result.scannedFilesCount(), + "deleted_files_count", result.deletedFilesCount()); } public void executeAddFiles(ConnectorSession session, IcebergTableExecuteHandle executeHandle) @@ -2240,17 +2250,21 @@ public void executeAddFilesFromTable(ConnectorSession session, IcebergTableExecu icebergScanExecutor); } - private void scanAndDeleteInvalidFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, Instant expiration, Set validFiles, Map fileIoProperties) + private ScanAndDeleteResult scanAndDeleteInvalidFiles(Table table, ConnectorSession session, SchemaTableName schemaTableName, Instant expiration, Set validFiles, Map fileIoProperties) { List> deleteFutures = new ArrayList<>(); + long scannedFilesCount = 0; + long deletedFilesCount = 0; try { List filesToDelete = new ArrayList<>(DELETE_BATCH_SIZE); TrinoFileSystem fileSystem = fileSystemFactory.create(session.getIdentity(), fileIoProperties); FileIterator allFiles = fileSystem.listFiles(Location.of(table.location())); while (allFiles.hasNext()) { FileEntry entry = allFiles.next(); + scannedFilesCount++; if (entry.lastModified().isBefore(expiration) && !validFiles.contains(entry.location().fileName())) { filesToDelete.add(entry.location()); + deletedFilesCount++; if (filesToDelete.size() >= DELETE_BATCH_SIZE) { List finalFilesToDelete = filesToDelete; deleteFutures.add(icebergFileDeleteExecutor.submit(() -> deleteFiles(finalFilesToDelete, schemaTableName, fileSystem))); @@ -2277,8 +2291,11 @@ private void scanAndDeleteInvalidFiles(Table table, ConnectorSession session, Sc // Ensure any futures still running are canceled in case of failure deleteFutures.forEach(future -> future.cancel(true)); } + return new ScanAndDeleteResult(scannedFilesCount, deletedFilesCount); } + private record ScanAndDeleteResult(long scannedFilesCount, long deletedFilesCount) {} + private void deleteFiles(List files, SchemaTableName schemaTableName, TrinoFileSystem fileSystem) { log.debug("Deleting files while removing orphan files for table %s [%s]", schemaTableName, files); @@ -2328,6 +2345,12 @@ public Optional getInfo(ConnectorSession session, ConnectorTableHandle t totalDeleteFiles)); } + @Override + public io.trino.spi.metrics.Metrics getMetrics(ConnectorSession session) + { + return catalog.getMetrics(); + } + @Override public void dropTable(ConnectorSession session, ConnectorTableHandle tableHandle) { @@ -2619,6 +2642,11 @@ private void dropField(ConnectorSession session, ConnectorTableHandle tableHandl if (isPartitionColumn) { throw new TrinoException(NOT_SUPPORTED, "Cannot drop partition field: " + name); } + boolean isSortColumn = icebergTable.sortOrder().fields().stream() + .anyMatch(field -> field.sourceId() == fieldId); + if (isSortColumn) { + throw new TrinoException(NOT_SUPPORTED, "Cannot drop sort field: " + name); + } int currentSpecId = icebergTable.spec().specId(); boolean columnUsedInOlderPartitionSpecs = icebergTable.specs().entrySet().stream() .filter(spec -> spec.getValue().specId() != currentSpecId) @@ -2812,12 +2840,16 @@ public void dropNotNullConstraint(ConnectorSession session, ConnectorTableHandle } @Override - public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean tableReplace) { if (!isExtendedStatisticsEnabled(session) || !isCollectExtendedStatisticsOnWrite(session)) { return TableStatisticsMetadata.empty(); } + if (tableReplace) { + return getStatisticsCollectionMetadata(tableMetadata, Optional.empty(), availableColumnNames -> {}); + } + ConnectorTableHandle tableHandle = getTableHandle(session, tableMetadata.getTable(), Optional.empty(), Optional.empty()); if (tableHandle == null) { // Assume new table (CTAS), collect NDV stats on all columns @@ -2852,6 +2884,12 @@ public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(Connector return getStatisticsCollectionMetadata(tableMetadata, Optional.of(columnsWithExtendedStatistics), availableColumnNames -> {}); } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) + { + throw new UnsupportedOperationException("This variant of getStatisticsCollectionMetadataForWrite is unsupported"); + } + @Override public ConnectorAnalyzeMetadata getStatisticsCollectionMetadata(ConnectorSession session, ConnectorTableHandle tableHandle, Map analyzeProperties) { @@ -3401,6 +3439,9 @@ else if (isMetadataColumnId(columnHandle.getId())) { private static List loadAllManifestsFromSnapshot(Table icebergTable, Snapshot snapshot) { + if (snapshot == null) { + return ImmutableList.of(); + } try { return snapshot.allManifests(icebergTable.io()); } @@ -3541,9 +3582,12 @@ public TableStatistics getTableStatistics(ConnectorSession session, ConnectorTab } IcebergTableHandle originalHandle = (IcebergTableHandle) tableHandle; + + if (originalHandle.isRecordScannedFiles()) { + return TableStatistics.empty(); + } // Certain table handle attributes are not applicable to select queries (which need stats). // If this changes, the caching logic may here may need to be revised. - checkArgument(!originalHandle.isRecordScannedFiles(), "Unexpected scanned files recording set"); checkArgument(originalHandle.getMaxScannedFileSize().isEmpty(), "Unexpected max scanned file size set"); IcebergTableHandle cacheKey = new IcebergTableHandle( diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java index 3432159270f5..872f12ca9501 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSink.java @@ -149,6 +149,7 @@ public IcebergPageSink( List sortOrder, DataSize sortingFileWriterBufferSize, int sortingFileWriterMaxOpenFiles, + Optional sortedWritingLocalStagingPath, TypeManager typeManager, PageSorter pageSorter) { @@ -171,13 +172,17 @@ public IcebergPageSink( this.sortedWritingEnabled = isSortedWritingEnabled(session); this.sortingFileWriterBufferSize = requireNonNull(sortingFileWriterBufferSize, "sortingFileWriterBufferSize is null"); this.sortingFileWriterMaxOpenFiles = sortingFileWriterMaxOpenFiles; - this.tempDirectory = Location.of(locationProvider.newDataLocation("trino-tmp-files")); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); this.columnTypes = getTopLevelColumns(outputSchema, typeManager).stream() .map(IcebergColumnHandle::getType) .collect(toImmutableList()); + this.tempDirectory = sortedWritingLocalStagingPath + .map(path -> path.replace("${USER}", session.getIdentity().getUser())) + .map(IcebergPageSink::createLocalSchemeIfAbsent) + .orElseGet(() -> Location.of(locationProvider.newDataLocation("trino-tmp-files"))); + if (sortedWritingEnabled) { ImmutableList.Builder sortColumnIndexes = ImmutableList.builder(); ImmutableList.Builder sortOrders = ImmutableList.builder(); @@ -591,6 +596,15 @@ private static int findFieldPosFromSchema(int fieldId, Types.StructType struct) throw new IllegalArgumentException("Could not find field " + fieldId + " in schema"); } + private static Location createLocalSchemeIfAbsent(String path) + { + Location location = Location.of(path); + if (location.scheme().isPresent()) { + return location; + } + return Location.of("local:///" + location.path()); + } + private static class WriteContext { private final IcebergFileWriter writer; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java index be6b8cb9dfa7..e94d1b5ba19e 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSinkProvider.java @@ -39,6 +39,7 @@ import org.apache.iceberg.io.LocationProvider; import java.util.Map; +import java.util.Optional; import static com.google.common.collect.Maps.transformValues; import static io.trino.plugin.iceberg.IcebergSessionProperties.maxPartitionsPerWriter; @@ -54,6 +55,7 @@ public class IcebergPageSinkProvider private final PageIndexerFactory pageIndexerFactory; private final DataSize sortingFileWriterBufferSize; private final int sortingFileWriterMaxOpenFiles; + private final Optional sortingFileWriterLocalStagingPath; private final TypeManager typeManager; private final PageSorter pageSorter; @@ -64,6 +66,7 @@ public IcebergPageSinkProvider( IcebergFileWriterFactory fileWriterFactory, PageIndexerFactory pageIndexerFactory, SortingFileWriterConfig sortingFileWriterConfig, + IcebergConfig icebergConfig, TypeManager typeManager, PageSorter pageSorter) { @@ -73,6 +76,7 @@ public IcebergPageSinkProvider( this.pageIndexerFactory = requireNonNull(pageIndexerFactory, "pageIndexerFactory is null"); this.sortingFileWriterBufferSize = sortingFileWriterConfig.getWriterSortBufferSize(); this.sortingFileWriterMaxOpenFiles = sortingFileWriterConfig.getMaxOpenSortFiles(); + this.sortingFileWriterLocalStagingPath = icebergConfig.getSortedWritingLocalStagingPath(); this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.pageSorter = requireNonNull(pageSorter, "pageSorter is null"); } @@ -111,6 +115,7 @@ private ConnectorPageSink createPageSink(ConnectorSession session, IcebergWritab tableHandle.sortOrder(), sortingFileWriterBufferSize, sortingFileWriterMaxOpenFiles, + sortingFileWriterLocalStagingPath, typeManager, pageSorter); } @@ -142,6 +147,7 @@ public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHa optimizeHandle.sortOrder(), sortingFileWriterBufferSize, sortingFileWriterMaxOpenFiles, + sortingFileWriterLocalStagingPath, typeManager, pageSorter); case OPTIMIZE_MANIFESTS: diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java index aad2add5ec40..56d266705723 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergPageSourceProvider.java @@ -574,7 +574,8 @@ private ReaderPageSourceWithRowPositions createDataPageSource( fileSchema, nameMapping, partition, - dataColumns); + dataColumns, + partitionKeys); }; } @@ -916,7 +917,7 @@ private static ReaderPageSourceWithRowPositions createParquetPageSource( ParquetDataSource dataSource = null; try { dataSource = createDataSource(inputFile, OptionalLong.of(fileSize), options, memoryContext, fileFormatDataSourceStats); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize(), Optional.empty()); FileMetadata fileMetaData = parquetMetadata.getFileMetaData(); MessageType fileSchema = fileMetaData.getSchema(); if (nameMapping.isPresent() && !ParquetSchemaUtil.hasIds(fileSchema)) { @@ -1027,6 +1028,7 @@ else if (!parquetIdToFieldName.containsKey(column.getBaseColumn().getId())) { options, exception -> handleException(dataSourceId, exception), Optional.empty(), + Optional.empty(), Optional.empty()); ConnectorPageSource pageSource = new ParquetPageSource(parquetReader); @@ -1112,7 +1114,8 @@ private static ReaderPageSourceWithRowPositions createAvroPageSource( Schema fileSchema, Optional nameMapping, String partition, - List columns) + List columns, + Map> partitionKeys) { InputFile file = new ForwardingInputFile(inputFile); OptionalLong fileModifiedTime = OptionalLong.empty(); @@ -1145,7 +1148,13 @@ private static ReaderPageSourceWithRowPositions createAvroPageSource( int nextOrdinal = 0; for (IcebergColumnHandle column : columns) { - if (column.isPartitionColumn()) { + if (partitionKeys.containsKey(column.getId())) { + Type trinoType = column.getType(); + transforms.constantValue(nativeValueToBlock( + trinoType, + deserializePartitionValue(trinoType, partitionKeys.get(column.getId()).orElse(null), column.getName()))); + } + else if (column.isPartitionColumn()) { transforms.constantValue(nativeValueToBlock(PARTITION.getType(), utf8Slice(partition))); } else if (column.isPathColumn()) { diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java index 1c7ba6ac43e3..b947b6cb179c 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergParquetFileWriter.java @@ -13,9 +13,11 @@ */ package io.trino.plugin.iceberg; +import com.google.common.collect.ImmutableList; import io.trino.filesystem.Location; import io.trino.filesystem.TrinoOutputFile; import io.trino.parquet.ParquetDataSourceId; +import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ParquetMetadata; import io.trino.parquet.writer.ParquetWriterOptions; import io.trino.plugin.hive.parquet.ParquetFileWriter; @@ -29,6 +31,8 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -36,7 +40,6 @@ import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_FILESYSTEM_ERROR; import static io.trino.plugin.iceberg.util.ParquetUtil.footerMetrics; -import static io.trino.plugin.iceberg.util.ParquetUtil.getSplitOffsets; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -83,7 +86,7 @@ public FileMetrics getFileMetrics() { ParquetMetadata parquetMetadata; try { - parquetMetadata = new ParquetMetadata(parquetFileWriter.getFileMetadata(), new ParquetDataSourceId(location.toString())); + parquetMetadata = new ParquetMetadata(parquetFileWriter.getFileMetadata(), new ParquetDataSourceId(location.toString()), Optional.empty()); return new FileMetrics(footerMetrics(parquetMetadata, Stream.empty(), metricsConfig), Optional.of(getSplitOffsets(parquetMetadata))); } catch (IOException | UncheckedIOException e) { @@ -126,4 +129,16 @@ public long getValidationCpuNanos() { return parquetFileWriter.getValidationCpuNanos(); } + + private static List getSplitOffsets(ParquetMetadata metadata) + throws IOException + { + List blocks = metadata.getBlocks(); + List splitOffsets = new ArrayList<>(blocks.size()); + for (BlockMetadata blockMetaData : blocks) { + splitOffsets.add(blockMetaData.columns().getFirst().getStartingPos()); + } + Collections.sort(splitOffsets); + return ImmutableList.copyOf(splitOffsets); + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSchemaProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSchemaProperties.java index bdc43ab1b9e3..714403fa5299 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSchemaProperties.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergSchemaProperties.java @@ -14,10 +14,12 @@ package io.trino.plugin.iceberg; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import io.trino.spi.session.PropertyMetadata; import java.util.List; +import java.util.Set; import static io.trino.spi.session.PropertyMetadata.stringProperty; @@ -25,6 +27,10 @@ public final class IcebergSchemaProperties { public static final String LOCATION_PROPERTY = "location"; + public static final Set SUPPORTED_SCHEMA_PROPERTIES = ImmutableSet.builder() + .add(LOCATION_PROPERTY) + .build(); + public final List> schemaProperties; @Inject diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/StructLikeWrapperWithFieldIdToIndex.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/StructLikeWrapperWithFieldIdToIndex.java index 4fa5c08f1a9c..14f3a18bceb0 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/StructLikeWrapperWithFieldIdToIndex.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/StructLikeWrapperWithFieldIdToIndex.java @@ -16,6 +16,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.StructLike; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructLikeWrapper; @@ -31,8 +32,12 @@ public class StructLikeWrapperWithFieldIdToIndex public static StructLikeWrapperWithFieldIdToIndex createStructLikeWrapper(FileScanTask fileScanTask) { - Types.StructType structType = fileScanTask.spec().partitionType(); - StructLikeWrapper partitionWrapper = StructLikeWrapper.forType(structType).set(fileScanTask.file().partition()); + return createStructLikeWrapper(fileScanTask.spec().partitionType(), fileScanTask.file().partition()); + } + + public static StructLikeWrapperWithFieldIdToIndex createStructLikeWrapper(Types.StructType structType, StructLike partition) + { + StructLikeWrapper partitionWrapper = StructLikeWrapper.forType(structType).set(partition); return new StructLikeWrapperWithFieldIdToIndex(partitionWrapper, structType); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableStatisticsWriter.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableStatisticsWriter.java index 69674474326c..56ce1fb3e3ba 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableStatisticsWriter.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/TableStatisticsWriter.java @@ -19,7 +19,7 @@ import com.google.common.graph.Traverser; import com.google.inject.Inject; import io.trino.plugin.base.io.ByteBuffers; -import io.trino.plugin.hive.NodeVersion; +import io.trino.spi.NodeVersion; import io.trino.spi.connector.ConnectorSession; import org.apache.datasketches.memory.Memory; import org.apache.datasketches.theta.CompactSketch; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java index 498a718b567a..e9a928d33cca 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/AbstractIcebergTableOperations.java @@ -240,7 +240,7 @@ protected void refreshFromMetadataLocation(String newLocation) { refreshFromMetadataLocation( newLocation, - metadataLocation -> TableMetadataParser.read(fileIo, fileIo.newInputFile(metadataLocation))); + metadataLocation -> TableMetadataParser.read(fileIo.newInputFile(metadataLocation))); } protected void refreshFromMetadataLocation(String newLocation, Function metadataLoader) diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java index 12d8419cbac3..823c8b8f9d79 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java @@ -24,6 +24,7 @@ import io.trino.spi.connector.RelationColumnsMetadata; import io.trino.spi.connector.RelationCommentMetadata; import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.metrics.Metrics; import io.trino.spi.security.TrinoPrincipal; import jakarta.annotation.Nullable; import org.apache.iceberg.BaseTable; @@ -200,4 +201,9 @@ void createMaterializedView( void updateColumnComment(ConnectorSession session, SchemaTableName schemaTableName, ColumnIdentity columnIdentity, Optional comment); Optional redirectTable(ConnectorSession session, SchemaTableName tableName, String hiveCatalogName); + + default Metrics getMetrics() + { + return Metrics.EMPTY; + } } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java index 16945ec42025..85d761d1062d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalog.java @@ -97,6 +97,7 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.function.UnaryOperator; +import java.util.regex.Pattern; import java.util.stream.Stream; import static com.google.common.base.MoreObjects.firstNonNull; @@ -166,6 +167,7 @@ public class TrinoGlueCatalog private static final Logger LOG = Logger.get(TrinoGlueCatalog.class); private static final int PER_QUERY_CACHES_SIZE = 1000; + private static final Pattern METADATA_PATTERN = Pattern.compile("/metadata/[^/]*$"); private final String trinoVersion; private final boolean cacheTableMetadata; @@ -290,9 +292,6 @@ public Map loadNamespaceMetadata(ConnectorSession session, Strin if (database.locationUri() != null) { metadata.put(LOCATION_PROPERTY, database.locationUri()); } - if (database.parameters() != null) { - metadata.putAll(database.parameters()); - } return metadata.buildOrThrow(); } catch (EntityNotFoundException e) { @@ -720,7 +719,7 @@ public void dropCorruptedTable(ConnectorSession session, SchemaTableName schemaT if (metadataLocation == null) { throw new TrinoException(ICEBERG_INVALID_METADATA, format("Table %s is missing [%s] property", schemaTableName, METADATA_LOCATION_PROP)); } - String tableLocation = metadataLocation.replaceFirst("/metadata/[^/]*$", ""); + String tableLocation = METADATA_PATTERN.matcher(metadataLocation).replaceFirst(""); deleteTableDirectory(fileSystemFactory.create(session), schemaTableName, tableLocation); invalidateTableCache(schemaTableName); } @@ -820,7 +819,7 @@ public void renameTable(ConnectorSession session, SchemaTableName from, SchemaTa if (metadataLocation == null) { throw new TrinoException(ICEBERG_INVALID_METADATA, format("Table %s is missing [%s] property", from, METADATA_LOCATION_PROP)); } - TableMetadata metadata = TableMetadataParser.read(io, io.newInputFile(metadataLocation)); + TableMetadata metadata = TableMetadataParser.read(io.newInputFile(metadataLocation)); TableInput tableInput = getTableInput( typeManager, to.getTableName(), diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java index 1aae3ab1a709..5b6b8e57ac25 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/TrinoGlueCatalogFactory.java @@ -16,7 +16,6 @@ import com.google.inject.Inject; import io.airlift.concurrent.BoundedExecutor; import io.trino.filesystem.TrinoFileSystemFactory; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; import io.trino.plugin.hive.security.UsingSystemSecurity; @@ -26,6 +25,7 @@ import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.plugin.iceberg.fileio.ForwardingFileIoFactory; +import io.trino.spi.NodeVersion; import io.trino.spi.catalog.CatalogName; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java index 79e20e863f90..06460d638b7e 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalog.java @@ -53,6 +53,7 @@ import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.TableNotFoundException; import io.trino.spi.connector.ViewNotFoundException; +import io.trino.spi.metrics.Metrics; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.type.TypeManager; import org.apache.iceberg.BaseTable; @@ -880,6 +881,12 @@ public Optional redirectTable(ConnectorSession session, return Optional.empty(); } + @Override + public Metrics getMetrics() + { + return metastore.getMetrics(); + } + @Override protected void invalidateTableCache(SchemaTableName schemaTableName) { diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java index 7eb40e177ef6..b8476a0a6601 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/hms/TrinoHiveCatalogFactory.java @@ -18,7 +18,6 @@ import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastoreFactory; import io.trino.metastore.cache.CachingHiveMetastore; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.TrinoViewHiveMetastore; import io.trino.plugin.hive.security.UsingSystemSecurity; import io.trino.plugin.iceberg.ForIcebergMetadata; @@ -27,6 +26,7 @@ import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.plugin.iceberg.fileio.ForwardingFileIoFactory; +import io.trino.spi.NodeVersion; import io.trino.spi.catalog.CatalogName; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java index 1c18177d5b4a..a18b0ac436eb 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/TrinoNessieCatalog.java @@ -59,9 +59,11 @@ import static com.google.common.base.Throwables.throwIfUnchecked; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.trino.cache.CacheUtils.uncheckedCacheGet; import static io.trino.filesystem.Locations.appendPath; import static io.trino.plugin.iceberg.IcebergSchemaProperties.LOCATION_PROPERTY; +import static io.trino.plugin.iceberg.IcebergSchemaProperties.SUPPORTED_SCHEMA_PROPERTIES; import static io.trino.plugin.iceberg.IcebergUtil.getIcebergTableWithMetadata; import static io.trino.plugin.iceberg.IcebergUtil.quotedTableName; import static io.trino.plugin.iceberg.catalog.nessie.IcebergNessieUtil.toIdentifier; @@ -125,7 +127,9 @@ public void dropNamespace(ConnectorSession session, String namespace) public Map loadNamespaceMetadata(ConnectorSession session, String namespace) { try { - return ImmutableMap.copyOf(nessieClient.loadNamespaceMetadata(Namespace.of(namespace))); + return nessieClient.loadNamespaceMetadata(Namespace.of(namespace)).entrySet().stream() + .filter(metadata -> SUPPORTED_SCHEMA_PROPERTIES.contains(metadata.getKey())) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); } catch (NoSuchNamespaceException e) { throw new SchemaNotFoundException(namespace); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java index 044070a8bc8d..46fc068c9f89 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java @@ -20,13 +20,13 @@ import com.google.inject.Inject; import io.airlift.units.Duration; import io.trino.cache.EvictableCacheBuilder; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergFileSystemFactory; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.plugin.iceberg.catalog.rest.IcebergRestCatalogConfig.SessionType; import io.trino.plugin.iceberg.fileio.ForwardingFileIoFactory; +import io.trino.spi.NodeVersion; import io.trino.spi.catalog.CatalogName; import io.trino.spi.security.ConnectorIdentity; import io.trino.spi.type.TypeManager; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java index 90467c027374..751dd20fe41d 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java @@ -26,7 +26,6 @@ import io.trino.cache.EvictableCacheBuilder; import io.trino.metastore.TableInfo; import io.trino.plugin.iceberg.ColumnIdentity; -import io.trino.plugin.iceberg.IcebergSchemaProperties; import io.trino.plugin.iceberg.IcebergUtil; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.catalog.rest.IcebergRestCatalogConfig.SessionType; @@ -85,11 +84,14 @@ import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static io.trino.cache.CacheUtils.uncheckedCacheGet; import static io.trino.filesystem.Locations.appendPath; import static io.trino.metastore.Table.TABLE_COMMENT; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_CATALOG_ERROR; import static io.trino.plugin.iceberg.IcebergErrorCode.ICEBERG_UNSUPPORTED_VIEW_DIALECT; +import static io.trino.plugin.iceberg.IcebergSchemaProperties.LOCATION_PROPERTY; +import static io.trino.plugin.iceberg.IcebergSchemaProperties.SUPPORTED_SCHEMA_PROPERTIES; import static io.trino.plugin.iceberg.IcebergUtil.quotedTableName; import static io.trino.plugin.iceberg.catalog.AbstractTrinoCatalog.ICEBERG_VIEW_RUN_AS_OWNER; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; @@ -221,7 +223,9 @@ public Map loadNamespaceMetadata(ConnectorSession session, Strin { try { // Return immutable metadata as direct modifications will not be reflected on the namespace - return ImmutableMap.copyOf(restSessionCatalog.loadNamespaceMetadata(convert(session), toRemoteNamespace(session, toNamespace(namespace)))); + return restSessionCatalog.loadNamespaceMetadata(convert(session), toRemoteNamespace(session, toNamespace(namespace))).entrySet().stream() + .filter(property -> SUPPORTED_SCHEMA_PROPERTIES.contains(property.getKey())) + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); } catch (NoSuchNamespaceException e) { throw new SchemaNotFoundException(namespace); @@ -576,7 +580,7 @@ public String defaultTableLocation(ConnectorSession session, SchemaTableName sch String tableName = createLocationForTable(schemaTableName.getTableName()); Map properties = loadNamespaceMetadata(session, schemaTableName.getSchemaName()); - String databaseLocation = (String) properties.get(IcebergSchemaProperties.LOCATION_PROPERTY); + String databaseLocation = (String) properties.get(LOCATION_PROPERTY); if (databaseLocation == null) { // Iceberg REST catalog doesn't require location property. // S3 Tables doesn't return the property. diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java index 630a5ca4f867..06e9d93b4bba 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/procedure/MigrationUtils.java @@ -154,7 +154,7 @@ private static Metrics parquetMetrics(TrinoInputFile file, MetricsConfig metrics { ParquetReaderOptions options = ParquetReaderOptions.defaultOptions(); try (ParquetDataSource dataSource = new TrinoParquetDataSource(file, ParquetReaderOptions.defaultOptions(), new FileFormatDataSourceStats())) { - ParquetMetadata metadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize()); + ParquetMetadata metadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize(), Optional.empty()); return ParquetUtil.footerMetrics(metadata, Stream.empty(), metricsConfig, nameMapping); } catch (IOException e) { diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/system/files/FilesTablePageSource.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/system/files/FilesTablePageSource.java index cba77dfab181..0c76b1a896f1 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/system/files/FilesTablePageSource.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/system/files/FilesTablePageSource.java @@ -17,6 +17,7 @@ import io.airlift.slice.Slices; import io.trino.filesystem.TrinoFileSystem; import io.trino.plugin.iceberg.IcebergUtil; +import io.trino.plugin.iceberg.StructLikeWrapperWithFieldIdToIndex; import io.trino.plugin.iceberg.fileio.ForwardingFileIoFactory; import io.trino.plugin.iceberg.system.FilesTable; import io.trino.plugin.iceberg.system.IcebergPartitionColumn; @@ -63,6 +64,7 @@ import static io.trino.plugin.iceberg.IcebergTypes.convertIcebergValueToTrino; import static io.trino.plugin.iceberg.IcebergUtil.primitiveFieldTypes; import static io.trino.plugin.iceberg.IcebergUtil.readerForManifest; +import static io.trino.plugin.iceberg.StructLikeWrapperWithFieldIdToIndex.createStructLikeWrapper; import static io.trino.plugin.iceberg.system.FilesTable.COLUMN_SIZES_COLUMN_NAME; import static io.trino.plugin.iceberg.system.FilesTable.CONTENT_COLUMN_NAME; import static io.trino.plugin.iceberg.system.FilesTable.EQUALITY_IDS_COLUMN_NAME; @@ -101,6 +103,7 @@ public final class FilesTablePageSource private final Schema schema; private final Schema metadataSchema; private final Map idToTypeMapping; + private final Map idToPartitionSpecMapping; private final List partitionFields; private final Optional partitionColumnType; private final List primitiveFields; @@ -123,15 +126,15 @@ public FilesTablePageSource( this.schema = SchemaParser.fromJson(requireNonNull(split.schemaJson(), "schema is null")); this.metadataSchema = SchemaParser.fromJson(requireNonNull(split.metadataTableJson(), "metadataSchema is null")); this.idToTypeMapping = primitiveFieldTypes(schema); - Map specs = split.partitionSpecsByIdJson().entrySet().stream().collect(toImmutableMap( + this.idToPartitionSpecMapping = split.partitionSpecsByIdJson().entrySet().stream().collect(toImmutableMap( Map.Entry::getKey, - entry -> PartitionSpecParser.fromJson(SchemaParser.fromJson(split.schemaJson()), entry.getValue()))); - this.partitionFields = getAllPartitionFields(schema, specs); + entry -> PartitionSpecParser.fromJson(schema, entry.getValue()))); + this.partitionFields = getAllPartitionFields(schema, idToPartitionSpecMapping); this.partitionColumnType = getPartitionColumnType(typeManager, partitionFields, schema); this.primitiveFields = IcebergUtil.primitiveFields(schema).stream() .sorted(Comparator.comparing(Types.NestedField::name)) .collect(toImmutableList()); - ManifestReader> manifestReader = closer.register(readerForManifest(split.manifestFile(), fileIoFactory.create(trinoFileSystem), specs)); + ManifestReader> manifestReader = closer.register(readerForManifest(split.manifestFile(), fileIoFactory.create(trinoFileSystem), idToPartitionSpecMapping)); // TODO figure out why selecting the specific column causes null to be returned for offset_splits this.contentIterator = closer.register(requireNonNull(manifestReader, "manifestReader is null").iterator()); this.pageBuilder = new PageBuilder(requiredColumns.stream().map(column -> { @@ -195,7 +198,12 @@ public SourcePage getNextSourcePage() writeValueOrNull(pageBuilder, SPEC_ID_COLUMN_NAME, contentFile::specId, INTEGER::writeInt); // partitions if (partitionColumnType.isPresent() && columnNameToIndex.containsKey(FilesTable.PARTITION_COLUMN_NAME)) { + PartitionSpec partitionSpec = idToPartitionSpecMapping.get(contentFile.specId()); + StructLikeWrapperWithFieldIdToIndex partitionStruct = createStructLikeWrapper(partitionSpec.partitionType(), contentFile.partition()); List partitionTypes = partitionTypes(partitionFields, idToTypeMapping); + List> partitionColumnClass = partitionTypes.stream() + .map(type -> type.typeId().javaClass()) + .collect(toImmutableList()); List partitionColumnTypes = partitionColumnType.orElseThrow().rowType().getFields().stream() .map(RowType.Field::getType) .collect(toImmutableList()); @@ -203,12 +211,13 @@ public SourcePage getNextSourcePage() if (pageBuilder.getBlockBuilder(columnNameToIndex.get(FilesTable.PARTITION_COLUMN_NAME)) instanceof RowBlockBuilder rowBlockBuilder) { rowBlockBuilder.buildEntry(fields -> { for (int i = 0; i < partitionColumnTypes.size(); i++) { - Type type = partitionTypes.get(i); io.trino.spi.type.Type trinoType = partitionColumnType.get().rowType().getFields().get(i).getType(); Object value = null; Integer fieldId = partitionColumnType.get().fieldIds().get(i); - if (fieldId != null) { - value = convertIcebergValueToTrino(type, contentFile.partition().get(i, type.typeId().javaClass())); + if (partitionStruct.getFieldIdToIndex().containsKey(fieldId)) { + value = convertIcebergValueToTrino( + partitionTypes.get(i), + partitionStruct.getStructLikeWrapper().get().get(partitionStruct.getFieldIdToIndex().get(fieldId), partitionColumnClass.get(i))); } writeNativeValue(trinoType, fields.get(i), value); } diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java index 98f50940b419..d25f3aaf6365 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/util/ParquetUtil.java @@ -14,8 +14,6 @@ package io.trino.plugin.iceberg.util; -import com.google.common.collect.ImmutableList; -import io.trino.parquet.ParquetCorruptionException; import io.trino.parquet.metadata.BlockMetadata; import io.trino.parquet.metadata.ColumnChunkMetadata; import io.trino.parquet.metadata.ParquetMetadata; @@ -41,13 +39,12 @@ import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -70,7 +67,7 @@ public final class ParquetUtil private ParquetUtil() {} public static Metrics footerMetrics(ParquetMetadata metadata, Stream> fieldMetrics, MetricsConfig metricsConfig) - throws ParquetCorruptionException + throws IOException { return footerMetrics(metadata, fieldMetrics, metricsConfig, null); } @@ -80,7 +77,7 @@ public static Metrics footerMetrics( Stream> fieldMetrics, MetricsConfig metricsConfig, NameMapping nameMapping) - throws ParquetCorruptionException + throws IOException { requireNonNull(fieldMetrics, "fieldMetrics should not be null"); @@ -158,18 +155,6 @@ public static Metrics footerMetrics( toBufferMap(fileSchema, upperBounds)); } - public static List getSplitOffsets(ParquetMetadata metadata) - throws ParquetCorruptionException - { - List blocks = metadata.getBlocks(); - List splitOffsets = new ArrayList<>(blocks.size()); - for (BlockMetadata blockMetaData : blocks) { - splitOffsets.add(blockMetaData.getStartingPos()); - } - Collections.sort(splitOffsets); - return ImmutableList.copyOf(splitOffsets); - } - private static void updateFromFieldMetrics( Map> idToFieldMetricsMap, MetricsConfig metricsConfig, diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java index 757c87682b54..b1c373b253bb 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergConnectorTest.java @@ -23,6 +23,8 @@ import io.airlift.units.DataSize; import io.airlift.units.Duration; import io.trino.Session; +import io.trino.connector.MockConnectorFactory; +import io.trino.connector.MockConnectorPlugin; import io.trino.execution.StageId; import io.trino.execution.StageInfo; import io.trino.execution.StagesInfo; @@ -42,8 +44,10 @@ import io.trino.spi.connector.ConstraintApplicationResult; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.connector.TableNotFoundException; +import io.trino.spi.metrics.Metrics; import io.trino.spi.predicate.Domain; import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.statistics.TableStatistics; import io.trino.sql.planner.Plan; import io.trino.sql.planner.optimizations.PlanNodeSearcher; import io.trino.sql.planner.plan.ExchangeNode; @@ -56,9 +60,11 @@ import io.trino.testing.DistributedQueryRunner; import io.trino.testing.MaterializedResult; import io.trino.testing.MaterializedRow; +import io.trino.testing.QueryFailedException; import io.trino.testing.QueryRunner; import io.trino.testing.QueryRunner.MaterializedResultWithPlan; import io.trino.testing.TestingConnectorBehavior; +import io.trino.testing.TestingSession; import io.trino.testing.sql.TestTable; import org.apache.avro.Schema; import org.apache.avro.file.DataFileReader; @@ -113,6 +119,8 @@ import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; import static io.trino.SystemSessionProperties.DETERMINE_PARTITION_COUNT_FOR_WRITE_ENABLED; import static io.trino.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; +import static io.trino.SystemSessionProperties.IGNORE_STATS_CALCULATOR_FAILURES; +import static io.trino.SystemSessionProperties.ITERATIVE_OPTIMIZER_TIMEOUT; import static io.trino.SystemSessionProperties.MAX_HASH_PARTITION_COUNT; import static io.trino.SystemSessionProperties.MAX_WRITER_TASK_COUNT; import static io.trino.SystemSessionProperties.SCALE_WRITERS; @@ -179,6 +187,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assertions.offset; +import static org.assertj.core.api.Fail.fail; import static org.junit.jupiter.api.Assumptions.abort; public abstract class BaseIcebergConnectorTest @@ -217,6 +226,29 @@ protected IcebergQueryRunner.Builder createQueryRunnerBuilder() .setInitialTables(REQUIRED_TPCH_TABLES); } + @BeforeAll + public void initMockMetricsCatalog() + { + QueryRunner queryRunner = getQueryRunner(); + String mockConnector = "mock_metrics"; + queryRunner.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder() + .withName(mockConnector) + .withListSchemaNames(_ -> ImmutableList.of("default")) + .withGetTableStatistics(_ -> { + try { + Thread.sleep(110); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + return TableStatistics.empty(); + }) + .build())); + + queryRunner.createCatalog("mock_metrics", mockConnector); + } + @BeforeAll public void initFileSystem() { @@ -246,6 +278,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_REPORTING_WRITTEN_BYTES -> true; case SUPPORTS_ADD_COLUMN_NOT_NULL_CONSTRAINT, SUPPORTS_DEFAULT_COLUMN_VALUE, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_REFRESH_VIEW, SUPPORTS_RENAME_MATERIALIZED_VIEW_ACROSS_SCHEMAS, SUPPORTS_TOPN_PUSHDOWN -> false; @@ -1631,7 +1664,7 @@ public void testDroppingSortColumn() "WITH (sorted_by = ARRAY['comment']) AS SELECT * FROM nation WITH NO DATA")) { assertUpdate(withSmallRowGroups, "INSERT INTO " + table.getName() + " SELECT * FROM nation", 25); assertThat(query("ALTER TABLE " + table.getName() + " DROP COLUMN comment")) - .failure().hasMessageContaining("Cannot find source column for sort field"); + .failure().hasMessageContaining("Cannot drop sort field: comment"); } } @@ -6613,6 +6646,34 @@ public void testExpireSnapshotsParameterValidation() "\\QRetention specified (33.00s) is shorter than the minimum retention configured in the system (7.00d). Minimum retention can be changed with iceberg.expire-snapshots.min-retention configuration property or iceberg.expire_snapshots_min_retention session property"); } + @Test + public void testExplainOptimize() + { + Session sessionWithIgnoreStatsCalculatorFailuresFalse = withoutIgnoreStatsCalculatorFailures(getSession()); + + String tableName = "test_explain_optimize" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + " (key varchar, value integer) WITH (partitioning = ARRAY['key'])"); + assertUpdate("INSERT INTO " + tableName + " VALUES ('one', 1)", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES ('two', 2)", 1); + + assertExplain(sessionWithIgnoreStatsCalculatorFailuresFalse, "EXPLAIN ALTER TABLE " + tableName + " EXECUTE OPTIMIZE", + ".*Output layout:.*"); + } + + @Test + public void testExplainAnalyzeOptimize() + { + Session sessionWithIgnoreStatsCalculatorFailuresFalse = withoutIgnoreStatsCalculatorFailures(getSession()); + + String tableName = "test_explain_analyze_optimize" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + " (key varchar, value integer) WITH (partitioning = ARRAY['key'])"); + assertUpdate("INSERT INTO " + tableName + " VALUES ('one', 1)", 1); + assertUpdate("INSERT INTO " + tableName + " VALUES ('two', 2)", 1); + + assertExplain(sessionWithIgnoreStatsCalculatorFailuresFalse, "EXPLAIN ANALYZE ALTER TABLE " + tableName + " EXECUTE OPTIMIZE", + ".*Output layout:.*"); + } + @Test public void testRemoveOrphanFilesWithUnexpectedMissingManifest() throws Exception @@ -6645,7 +6706,10 @@ public void testRemoveOrphanFiles() List initialDataFiles = getAllDataFilesFromTableDirectory(tableName); assertThat(initialDataFiles).contains(orphanFile); - assertQuerySucceeds(sessionWithShortRetentionUnlocked, "ALTER TABLE " + tableName + " EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')"); + assertUpdate( + sessionWithShortRetentionUnlocked, + "ALTER TABLE " + tableName + " EXECUTE REMOVE_ORPHAN_FILES (retention_threshold => '0s')", + "VALUES ('processed_manifests_count', 3), ('active_files_count', 16), ('scanned_files_count', 17), ('deleted_files_count', 1)"); assertQuery("SELECT * FROM " + tableName, "VALUES ('one', 1), ('three', 3)"); List updatedDataFiles = getAllDataFilesFromTableDirectory(tableName); @@ -8490,6 +8554,11 @@ public void testBucketedSelect() expectedQuery = "SELECT a.custkey, b.orderkey FROM orders a JOIN orders b on a.orderkey = b.custkey"; assertQuery(planWithTableNodePartitioning, query, expectedQuery, assertRemoteExchangesCount(1)); assertQuery(planWithoutTableNodePartitioning, query, expectedQuery, assertRemoteExchangesCount(2)); + + // optimize should not require a remote exchange between the scan and the execute nodes + query = "ALTER TABLE test_bucketed_select EXECUTE OPTIMIZE"; + assertUpdate(planWithTableNodePartitioning, query, assertRemoteExchangesCount(1)); + assertUpdate(planWithoutTableNodePartitioning, query, assertRemoteExchangesCount(2)); } finally { assertUpdate("DROP TABLE IF EXISTS test_bucketed_select"); @@ -9214,6 +9283,38 @@ public void testCreateTableWithDataLocationButObjectStoreLayoutDisabled() "Data location can only be set when object store layout is enabled"); } + @Test + public void testCatalogMetadataMetrics() + { + MaterializedResultWithPlan result = getQueryRunner().executeWithPlan( + getSession(), + "SELECT count(*) FROM region r, nation n WHERE r.regionkey = n.regionkey"); + Map metrics = getCatalogMetadataMetrics(result.queryId()); + assertCountMetricExists(metrics, "iceberg", "metastore.all.time.total"); + assertDistributionMetricExists(metrics, "iceberg", "metastore.all.time.distribution"); + assertCountMetricExists(metrics, "iceberg", "metastore.getTable.time.total"); + assertDistributionMetricExists(metrics, "iceberg", "metastore.getTable.time.distribution"); + } + + @Test + public void testCatalogMetadataMetricsWithOptimizerTimeoutExceeded() + { + String query = "SELECT count(*) FROM region r, nation n, mock_metrics.default.mock_table m WHERE r.regionkey = n.regionkey"; + try { + Session smallOptimizerTimeout = TestingSession.testSessionBuilder(getSession()) + .setSystemProperty(ITERATIVE_OPTIMIZER_TIMEOUT, "100ms") + .build(); + MaterializedResultWithPlan result = getQueryRunner().executeWithPlan(smallOptimizerTimeout, query); + fail(format("Expected query to fail: %s [QueryId: %s]", query, result.queryId())); + } + catch (QueryFailedException e) { + assertThat(e.getMessage()).contains("The optimizer exhausted the time limit"); + Map metrics = getCatalogMetadataMetrics(e.getQueryId()); + assertCountMetricExists(metrics, "iceberg", "metastore.all.time.total"); + assertCountMetricExists(metrics, "iceberg", "metastore.getTable.time.total"); + } + } + @Test @Override public void testSetFieldMapKeyType() @@ -9321,6 +9422,13 @@ protected Session withoutSmallFileThreshold(Session session) .build(); } + private Session withoutIgnoreStatsCalculatorFailures(Session session) + { + return Session.builder(session) + .setSystemProperty(IGNORE_STATS_CALCULATOR_FAILURES, "false") + .build(); + } + private Session withSingleWriterPerTask(Session session) { return Session.builder(session) @@ -9449,7 +9557,7 @@ private QueryId executeWithQueryId(String sql) private void assertQueryIdAndUserStored(String tableName, QueryId queryId) { assertThat(getFieldFromLatestSnapshotSummary(tableName, TRINO_QUERY_ID_NAME)) - .isEqualTo(queryId.toString()); + .isEqualTo(queryId.id()); assertThat(getFieldFromLatestSnapshotSummary(tableName, TRINO_USER_NAME)) .isEqualTo("user"); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java index d840314fe501..f36f4c71fbe9 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/BaseIcebergMaterializedViewTest.java @@ -764,7 +764,7 @@ public void testMaterializedViewSnapshotSummariesHaveTrinoQueryId() .executeWithPlan(getSession(), format("REFRESH MATERIALIZED VIEW %s", materializedViewName)) .queryId(); String savedQueryId = getStorageTableMetadata(materializedViewName).currentSnapshot().summary().get("trino_query_id"); - assertThat(savedQueryId).isEqualTo(refreshQueryId.getId()); + assertThat(savedQueryId).isEqualTo(refreshQueryId.id()); } finally { assertUpdate("DROP TABLE " + sourceTableName); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java index 3b0e92e9a877..bf5d3082f670 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergQueryRunner.java @@ -21,6 +21,7 @@ import io.airlift.log.Logger; import io.airlift.log.Logging; import io.trino.plugin.exchange.filesystem.FileSystemExchangePlugin; +import io.trino.plugin.hive.TestingHivePlugin; import io.trino.plugin.hive.containers.Hive3MinioDataLake; import io.trino.plugin.hive.containers.HiveHadoop; import io.trino.plugin.iceberg.catalog.jdbc.TestingIcebergJdbcServer; @@ -549,6 +550,10 @@ public static void main(String[] args) .addIcebergProperty("hive.metastore.catalog.dir", metastoreDir.toURI().toString()) .setInitialTables(TpchTable.getTables()) .build(); + queryRunner.installPlugin(new TestingHivePlugin(metastoreDir.toPath())); + queryRunner.createCatalog("hive", "hive", ImmutableMap.builder() + .put("hive.security", "allow-all") + .buildOrThrow()); log.info("======== SERVER STARTED ========"); log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java index 826a3d01eeb0..5b80c3494fea 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/IcebergTestUtils.java @@ -257,7 +257,7 @@ public static Map getMetadataFileAndUpdatedMillis(TrinoFileSystem while (fileIterator.hasNext()) { FileEntry entry = fileIterator.next(); if (fromFilePath(entry.location().path()) == METADATA_JSON) { - TableMetadata tableMetadata = TableMetadataParser.read(null, new ForwardingInputFile(trinoFileSystem.newInputFile(entry.location()))); + TableMetadata tableMetadata = TableMetadataParser.read(new ForwardingInputFile(trinoFileSystem.newInputFile(entry.location()))); metadataFiles.put(entry.location().path(), tableMetadata.lastUpdatedMillis()); } } @@ -267,7 +267,7 @@ public static Map getMetadataFileAndUpdatedMillis(TrinoFileSystem public static ParquetMetadata getParquetFileMetadata(TrinoInputFile inputFile) { try (TrinoParquetDataSource dataSource = new TrinoParquetDataSource(inputFile, ParquetReaderOptions.defaultOptions(), new FileFormatDataSourceStats())) { - return MetadataReader.readFooter(dataSource); + return MetadataReader.readFooter(dataSource, Optional.empty()); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java index d7410a93f069..f7d9d02e94b6 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergConfig.java @@ -20,6 +20,7 @@ import io.airlift.units.Duration; import io.trino.plugin.hive.HiveCompressionOption; import jakarta.validation.constraints.AssertFalse; +import jakarta.validation.constraints.AssertTrue; import org.junit.jupiter.api.Test; import java.util.Map; @@ -70,6 +71,7 @@ public void testDefaults() .setRegisterTableProcedureEnabled(false) .setAddFilesProcedureEnabled(false) .setSortedWritingEnabled(true) + .setSortedWritingLocalStagingPath(null) .setQueryPartitionFilterRequired(false) .setQueryPartitionFilterRequiredSchemas(ImmutableSet.of()) .setSplitManagerThreads(Integer.toString(Math.min(Runtime.getRuntime().availableProcessors() * 2, 32))) @@ -114,6 +116,7 @@ public void testExplicitPropertyMappings() .put("iceberg.register-table-procedure.enabled", "true") .put("iceberg.add-files-procedure.enabled", "true") .put("iceberg.sorted-writing-enabled", "false") + .put("iceberg.sorted-writing.local-staging-path", "/tmp/trino") .put("iceberg.query-partition-filter-required", "true") .put("iceberg.query-partition-filter-required-schemas", "bronze,silver") .put("iceberg.split-manager-threads", "42") @@ -154,6 +157,7 @@ public void testExplicitPropertyMappings() .setRegisterTableProcedureEnabled(true) .setAddFilesProcedureEnabled(true) .setSortedWritingEnabled(false) + .setSortedWritingLocalStagingPath("/tmp/trino") .setQueryPartitionFilterRequired(true) .setQueryPartitionFilterRequiredSchemas(ImmutableSet.of("bronze", "silver")) .setSplitManagerThreads("42") @@ -181,5 +185,12 @@ public void testValidation() "storageSchemaSetWhenHidingIsEnabled", "iceberg.materialized-views.storage-schema may only be set when iceberg.materialized-views.hide-storage-table is set to false", AssertFalse.class); + + assertFailsValidation( + new IcebergConfig() + .setSortedWritingLocalStagingPath("s3://bucket/path"), + "sortedWritingLocalStagingPathValid", + "iceberg.sorted-writing.local-staging-path must not use any prefix other than file:// or local://", + AssertTrue.class); } } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPartitionEvolution.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPartitionEvolution.java index e6a84857053f..07119664b089 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPartitionEvolution.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergPartitionEvolution.java @@ -14,6 +14,7 @@ package io.trino.plugin.iceberg; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.MaterializedRow; import io.trino.testing.QueryRunner; @@ -39,6 +40,33 @@ protected QueryRunner createQueryRunner() .build(); } + @Test + public void testSelectPartition() + { + String tableName = "test_select_partition_from_files" + randomNameSuffix(); + assertUpdate("CREATE TABLE " + tableName + " WITH (partitioning = ARRAY['regionkey', 'truncate(name, 1)']) AS SELECT * FROM nation WHERE nationkey <= 2", 3); + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES partitioning = DEFAULT"); + assertUpdate("INSERT INTO " + tableName + " SELECT * FROM nation WHERE nationkey <= 2", 3); + assertUpdate("ALTER TABLE " + tableName + " SET PROPERTIES partitioning = ARRAY['nationkey', 'regionkey', 'truncate(name, 1)']"); + assertUpdate("INSERT INTO " + tableName + " SELECT * FROM nation WHERE nationkey <= 2", 3); + + List files = computeActual("SELECT partition FROM \"" + tableName + "$files\"").getMaterializedRows(); + assertThat(files).hasSize(7); + List> partitionColumn = files.stream().map(materializedRow -> ((MaterializedRow) materializedRow.getField(0)).getFields()).toList(); + assertThat(partitionColumn).containsExactlyInAnyOrder( + // ['regionkey', 'truncate(name, 1)'] + Lists.newArrayList(1L, "B", null), + Lists.newArrayList(0L, "A", null), + Lists.newArrayList(1L, "A", null), + // DEFAULT + Lists.newArrayList(null, null, null), + // ['nationkey', 'regionkey', 'truncate(name, 1)'] + Lists.newArrayList(1L, "B", 2L), + Lists.newArrayList(0L, "A", 0L), + Lists.newArrayList(1L, "A", 1L)); + assertUpdate("DROP TABLE " + tableName); + } + @Test public void testRemovePartitioning() { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSortedWriting.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSortedWriting.java new file mode 100644 index 000000000000..7df259125e7c --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergSortedWriting.java @@ -0,0 +1,97 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg; + +import com.google.common.collect.ImmutableList; +import io.trino.Session; +import io.trino.filesystem.Location; +import io.trino.filesystem.TrinoFileSystem; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import io.trino.testing.sql.TestTable; +import io.trino.tpch.TpchTable; +import org.apache.iceberg.FileFormat; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import static io.trino.plugin.iceberg.IcebergTestUtils.checkOrcFileSorting; +import static io.trino.plugin.iceberg.IcebergTestUtils.checkParquetFileSorting; +import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; +import static io.trino.testing.TestingConnectorSession.SESSION; +import static org.apache.iceberg.FileFormat.PARQUET; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestIcebergSortedWriting + extends AbstractTestQueryFramework +{ + private TrinoFileSystem fileSystem; + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return IcebergQueryRunner.builder() + .setInitialTables(ImmutableList.of(TpchTable.LINE_ITEM)) + .addIcebergProperty("iceberg.sorted-writing-enabled", "true") + // Test staging of sorted writes to local disk + .addIcebergProperty("iceberg.sorted-writing.local-staging-path", "/tmp/trino-${USER}") + // Allows testing the sorting writer flushing to the file system with smaller tables + .addIcebergProperty("iceberg.writer-sort-buffer-size", "1MB") + .build(); + } + + @BeforeAll + public void initFileSystem() + { + fileSystem = getFileSystemFactory(getDistributedQueryRunner()).create(SESSION); + } + + @Test + public void testSortedWritingWithLocalStaging() + { + testSortedWritingWithLocalStaging(FileFormat.ORC); + testSortedWritingWithLocalStaging(FileFormat.PARQUET); + } + + private void testSortedWritingWithLocalStaging(FileFormat format) + { + // Using a larger table forces buffered data to be written to disk + Session withSmallRowGroups = Session.builder(getSession()) + .setCatalogSessionProperty("iceberg", "orc_writer_max_stripe_rows", "200") + .setCatalogSessionProperty("iceberg", "parquet_writer_block_size", "20kB") + .setCatalogSessionProperty("iceberg", "parquet_writer_batch_size", "200") + .build(); + try (TestTable table = new TestTable( + getQueryRunner()::execute, + "test_sorted_lineitem_table", + "WITH (sorted_by = ARRAY['comment'], format = '" + format.name() + "') AS TABLE tpch.tiny.lineitem WITH NO DATA")) { + assertUpdate( + withSmallRowGroups, + "INSERT INTO " + table.getName() + " TABLE tpch.tiny.lineitem", + "VALUES 60175"); + for (Object filePath : computeActual("SELECT file_path from \"" + table.getName() + "$files\"").getOnlyColumnAsSet()) { + assertThat(isFileSorted(Location.of((String) filePath), "comment", format)).isTrue(); + } + assertQuery("SELECT * FROM " + table.getName(), "SELECT * FROM lineitem"); + } + } + + private boolean isFileSorted(Location path, String sortColumnName, FileFormat format) + { + if (format == PARQUET) { + return checkParquetFileSorting(fileSystem.newInputFile(path), sortColumnName); + } + return checkOrcFileSorting(fileSystem, path, sortColumnName); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java index aaa5af59e926..86ecbe4bde31 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergStatistics.java @@ -18,6 +18,7 @@ import io.trino.Session; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.QueryRunner; +import io.trino.testing.sql.TestTable; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -972,6 +973,31 @@ public void testShowStatsAsOf() assertUpdate("DROP TABLE show_stats_as_of"); } + @Test + public void testShowStatsReplaceTable() + { + try (TestTable table = newTrinoTable("show_stats_after_replace_table_", "AS SELECT 1 a, 2 b")) { + assertThat(query("SHOW STATS FOR " + table.getName())) + .skippingTypesCheck() + .matches(""" + VALUES + ('a', null, 1e0, 0e0, NULL, '1', '1'), + ('b', null, 1e0, 0e0, NULL, '2', '2'), + (NULL, NULL, NULL, NULL, 1e0, NULL, NULL) + """); + + assertUpdate("CREATE OR REPLACE TABLE " + table.getName() + " AS SELECT 3 x, 4 y", 1); + assertThat(query("SHOW STATS FOR " + table.getName())) + .skippingTypesCheck() + .matches(""" + VALUES + ('x', null, 1e0, 0e0, NULL, '3', '3'), + ('y', null, 1e0, 0e0, NULL, '4', '4'), + (NULL, NULL, NULL, NULL, 1e0, NULL, NULL) + """); + } + } + @Test public void testShowStatsAfterExpiration() { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java index b8da11f5e764..9b6d9d4f8533 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java @@ -1494,7 +1494,7 @@ void testAnalyzeNoSnapshot() catalog.newCreateTableTransaction( SESSION, schemaTableName, - new Schema(Types.NestedField.of(1, true, "x", Types.LongType.get())), + new Schema(Types.NestedField.optional(1, "x", Types.LongType.get())), PartitionSpec.unpartitioned(), SortOrder.unsorted(), Optional.ofNullable(catalog.defaultTableLocation(SESSION, schemaTableName)), diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java index ebac31ac9970..212d9c282390 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/BaseTrinoCatalogTest.java @@ -19,7 +19,6 @@ import io.trino.metastore.TableInfo; import io.trino.metastore.TableInfo.ExtendedRelationType; import io.trino.plugin.base.util.AutoCloseableCloser; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.orc.OrcReaderConfig; import io.trino.plugin.hive.orc.OrcWriterConfig; import io.trino.plugin.hive.parquet.ParquetReaderConfig; @@ -30,6 +29,7 @@ import io.trino.plugin.iceberg.IcebergMetadata; import io.trino.plugin.iceberg.IcebergSessionProperties; import io.trino.plugin.iceberg.TableStatisticsWriter; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorMetadata; @@ -167,6 +167,40 @@ public void testNonLowercaseNamespace() } } + @Test + public void testSchemaWithInvalidProperties() + throws Exception + { + String namespace = "test_schema_invalid_properties" + randomNameSuffix(); + + TrinoCatalog catalog = createTrinoCatalog(false); + createNamespaceWithProperties(catalog, namespace, ImmutableMap.of("invalid_property", "test-value")); + try { + ConnectorMetadata icebergMetadata = new IcebergMetadata( + PLANNER_CONTEXT.getTypeManager(), + jsonCodec(CommitTaskData.class), + catalog, + (_, _) -> { + throw new UnsupportedOperationException(); + }, + TABLE_STATISTICS_READER, + new TableStatisticsWriter(new NodeVersion("test-version")), + Optional.empty(), + false, + _ -> false, + newDirectExecutorService(), + directExecutor(), + newDirectExecutorService(), + newDirectExecutorService()); + + assertThat(icebergMetadata.getSchemaProperties(SESSION, namespace)) + .doesNotContainKey("invalid_property"); + } + finally { + catalog.dropNamespace(SESSION, namespace); + } + } + @Test public void testCreateTable() throws Exception @@ -536,6 +570,8 @@ public void testListTables() } } + protected abstract void createNamespaceWithProperties(TrinoCatalog catalog, String namespace, Map properties); + protected void createMaterializedView( ConnectorSession session, TrinoCatalog catalog, diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestIcebergFileMetastoreCreateTableFailure.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestIcebergFileMetastoreCreateTableFailure.java index 3196539ce7a7..5f445fe71803 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestIcebergFileMetastoreCreateTableFailure.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestIcebergFileMetastoreCreateTableFailure.java @@ -18,11 +18,11 @@ import io.trino.metastore.HiveMetastore; import io.trino.metastore.PrincipalPrivileges; import io.trino.metastore.Table; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.HiveMetastoreConfig; import io.trino.plugin.hive.metastore.file.FileHiveMetastore; import io.trino.plugin.hive.metastore.file.FileHiveMetastoreConfig; import io.trino.plugin.iceberg.TestingIcebergPlugin; +import io.trino.spi.NodeVersion; import io.trino.spi.connector.SchemaNotFoundException; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java index 344330154420..2670ec2e1bce 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestTrinoHiveCatalogWithFileMetastore.java @@ -19,6 +19,7 @@ import io.trino.filesystem.Location; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.local.LocalFileSystemFactory; +import io.trino.metastore.Database; import io.trino.metastore.HiveMetastore; import io.trino.metastore.cache.CachingHiveMetastore; import io.trino.plugin.hive.TrinoViewHiveMetastore; @@ -43,6 +44,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Map; import java.util.Optional; import static com.google.common.io.MoreFiles.deleteRecursively; @@ -89,6 +91,17 @@ public void tearDown() deleteRecursively(tempDir, ALLOW_INSECURE); } + @Override + protected void createNamespaceWithProperties(TrinoCatalog catalog, String namespace, Map properties) + { + metastore.createDatabase(Database.builder() + .setDatabaseName(namespace) + .setOwnerName(Optional.of("test")) + .setOwnerType(Optional.of(PrincipalType.USER)) + .setParameters(properties) + .build()); + } + @Override protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java index d88c0ac316be..e773cc91f5ba 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/file/TestingIcebergFileMetastoreCatalogModule.java @@ -44,7 +44,7 @@ public TestingIcebergFileMetastoreCatalogModule(HiveMetastore metastore) @Override protected void setup(Binder binder) { - binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore)); + binder.bind(HiveMetastoreFactory.class).annotatedWith(RawHiveMetastoreFactory.class).toInstance(HiveMetastoreFactory.ofInstance(metastore, false)); install(new CachingHiveMetastoreModule()); binder.bind(IcebergTableOperationsProvider.class).to(FileMetastoreTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(TrinoCatalogFactory.class).to(TrinoHiveCatalogFactory.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java index 15c1b3e54872..cce193787a47 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogSkipArchive.java @@ -120,7 +120,7 @@ public void testNotRemoveExistingArchive() Map tableParameters = new HashMap<>(glueTable.parameters()); String metadataLocation = tableParameters.remove(METADATA_LOCATION_PROP); FileIO io = FILE_IO_FACTORY.create(getFileSystemFactory(getDistributedQueryRunner()).create(SESSION)); - TableMetadata metadata = TableMetadataParser.read(io, io.newInputFile(metadataLocation)); + TableMetadata metadata = TableMetadataParser.read(io.newInputFile(metadataLocation)); boolean cacheTableMetadata = new IcebergGlueCatalogConfig().isCacheTableMetadata(); TableInput tableInput = getTableInput(TESTING_TYPE_MANAGER, table.getName(), Optional.empty(), metadata, metadata.location(), metadataLocation, tableParameters, cacheTableMetadata); glueClient.updateTable(builder -> builder.databaseName(schemaName).tableInput(tableInput)); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java index 79400318e0be..205953755646 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/glue/TestTrinoGlueCatalog.java @@ -18,7 +18,6 @@ import io.airlift.log.Logger; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats; import io.trino.plugin.iceberg.CommitTaskData; import io.trino.plugin.iceberg.IcebergConfig; @@ -26,6 +25,7 @@ import io.trino.plugin.iceberg.TableStatisticsWriter; import io.trino.plugin.iceberg.catalog.BaseTrinoCatalogTest; import io.trino.plugin.iceberg.catalog.TrinoCatalog; +import io.trino.spi.NodeVersion; import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.ConnectorMaterializedViewDefinition; import io.trino.spi.connector.ConnectorMetadata; @@ -74,6 +74,15 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) return createGlueTrinoCatalog(useUniqueTableLocations, false); } + @Override + protected void createNamespaceWithProperties(TrinoCatalog catalog, String namespace, Map properties) + { + try (GlueClient glueClient = GlueClient.create()) { + glueClient.createDatabase(database -> database + .databaseInput(input -> input.name(namespace).parameters(properties))); + } + } + private TrinoCatalog createGlueTrinoCatalog(boolean useUniqueTableLocations, boolean useSystemSecurity) { GlueClient glueClient = GlueClient.create(); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java index db7166598d3a..0cd2880fb8f8 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/hms/TestTrinoHiveCatalogWithHiveMetastore.java @@ -24,6 +24,7 @@ import io.trino.filesystem.s3.S3FileSystemConfig; import io.trino.filesystem.s3.S3FileSystemFactory; import io.trino.filesystem.s3.S3FileSystemStats; +import io.trino.metastore.Database; import io.trino.metastore.Table; import io.trino.metastore.TableInfo; import io.trino.metastore.cache.CachingHiveMetastore; @@ -67,6 +68,7 @@ import static io.trino.metastore.cache.CachingHiveMetastore.createPerTransactionCache; import static io.trino.plugin.hive.TestingThriftHiveMetastoreBuilder.testingThriftHiveMetastoreBuilder; import static io.trino.plugin.hive.containers.HiveHadoop.HIVE3_IMAGE; +import static io.trino.plugin.hive.metastore.thrift.ThriftMetastoreUtil.toMetastoreApiDatabase; import static io.trino.plugin.iceberg.IcebergFileFormat.PARQUET; import static io.trino.plugin.iceberg.IcebergTableProperties.FILE_FORMAT_PROPERTY; import static io.trino.plugin.iceberg.IcebergTableProperties.FORMAT_VERSION_PROPERTY; @@ -119,6 +121,20 @@ public void tearDown() closer.close(); } + @Override + protected void createNamespaceWithProperties(TrinoCatalog catalog, String namespace, Map properties) + { + ThriftMetastore thriftMetastore = testingThriftHiveMetastoreBuilder() + .metastoreClient(dataLake.getHiveMetastoreEndpoint()) + .build(closer::register); + thriftMetastore.createDatabase(toMetastoreApiDatabase(Database.builder() + .setDatabaseName(namespace) + .setOwnerName(Optional.of("test")) + .setOwnerType(Optional.of(PrincipalType.USER)) + .setParameters(properties) + .build())); + } + @Override protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestingIcebergJdbcServer.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestingIcebergJdbcServer.java index 6e797a672034..670c0bc6383f 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestingIcebergJdbcServer.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/jdbc/TestingIcebergJdbcServer.java @@ -15,7 +15,7 @@ import org.apache.iceberg.jdbc.TestingTrinoIcebergJdbcUtil; import org.intellij.lang.annotations.Language; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import java.io.Closeable; import java.sql.Connection; @@ -24,7 +24,7 @@ import java.sql.Statement; import static java.lang.String.format; -import static org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT; +import static org.testcontainers.postgresql.PostgreSQLContainer.POSTGRESQL_PORT; public class TestingIcebergJdbcServer implements Closeable @@ -33,12 +33,12 @@ public class TestingIcebergJdbcServer public static final String PASSWORD = "test"; private static final String DATABASE = "tpch"; - private final PostgreSQLContainer dockerContainer; + private final PostgreSQLContainer dockerContainer; public TestingIcebergJdbcServer() { // TODO: Use Iceberg docker image once the community provides it - dockerContainer = new PostgreSQLContainer<>("postgres:12.10") + dockerContainer = new PostgreSQLContainer("postgres:12.10") .withDatabaseName(DATABASE) .withUsername(USER) .withPassword(PASSWORD); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java index d69d40db4bd1..a9f793998ee5 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java @@ -16,19 +16,20 @@ import com.google.common.collect.ImmutableMap; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.filesystem.hdfs.HdfsFileSystemFactory; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.iceberg.CommitTaskData; import io.trino.plugin.iceberg.IcebergMetadata; import io.trino.plugin.iceberg.TableStatisticsWriter; import io.trino.plugin.iceberg.catalog.BaseTrinoCatalogTest; import io.trino.plugin.iceberg.catalog.TrinoCatalog; import io.trino.plugin.iceberg.containers.NessieContainer; +import io.trino.spi.NodeVersion; import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.PrincipalType; import io.trino.spi.security.TrinoPrincipal; import io.trino.spi.type.TestingTypeManager; +import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.nessie.NessieIcebergClient; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -84,6 +85,18 @@ public void teardownServer() } } + @Override + protected void createNamespaceWithProperties(TrinoCatalog catalog, String namespace, Map properties) + { + IcebergNessieCatalogConfig icebergNessieCatalogConfig = new IcebergNessieCatalogConfig() + .setServerUri(URI.create(nessieContainer.getRestApiUri())); + NessieApiV2 nessieApi = NessieClientBuilder.createClientBuilderFromSystemSettings() + .withUri(nessieContainer.getRestApiUri()) + .build(NessieApiV2.class); + NessieIcebergClient nessieClient = new NessieIcebergClient(nessieApi, icebergNessieCatalogConfig.getDefaultReferenceName(), null, ImmutableMap.of()); + nessieClient.createNamespace(Namespace.of(namespace), properties); + } + @Override protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java index c4d4167a9e42..e4932b05977d 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java @@ -24,6 +24,7 @@ import io.trino.testing.QueryRunner; import io.trino.testing.TestingConnectorBehavior; import org.apache.iceberg.BaseTable; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -292,4 +293,14 @@ public void testRepeatUnregisterTable() {} @Test @Override // The procedure is unsupported in S3 Tables public void testUnregisterTableAccessControl() {} + + @Test + @Override + @Disabled // TODO https://github.com/trinodb/trino/issues/25129 Fix flaky test + public void testSelectInformationSchemaTables() {} + + @Test + @Override + @Disabled // TODO https://github.com/trinodb/trino/issues/25129 Fix flaky test + public void testIcebergTablesSystemTable() {} } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java index 884a1f889c65..ea86015a3bfb 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestTrinoRestCatalog.java @@ -16,12 +16,12 @@ import com.google.common.collect.ImmutableMap; import io.trino.cache.EvictableCacheBuilder; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.iceberg.CommitTaskData; import io.trino.plugin.iceberg.IcebergMetadata; import io.trino.plugin.iceberg.TableStatisticsWriter; import io.trino.plugin.iceberg.catalog.BaseTrinoCatalogTest; import io.trino.plugin.iceberg.catalog.TrinoCatalog; +import io.trino.spi.NodeVersion; import io.trino.spi.TrinoException; import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.ConnectorMetadata; @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Optional; +import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; import static io.airlift.json.JsonCodec.jsonCodec; @@ -62,6 +63,17 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) return createTrinoRestCatalog(useUniqueTableLocations, ImmutableMap.of()); } + @Override + protected void createNamespaceWithProperties(TrinoCatalog catalog, String namespace, Map properties) + { + catalog.createNamespace( + SESSION, + namespace, + properties.entrySet().stream() + .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)), + new TrinoPrincipal(PrincipalType.USER, SESSION.getUser())); + } + private static TrinoRestCatalog createTrinoRestCatalog(boolean useUniqueTableLocations, Map properties) throws IOException { diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java index 0b2f762d1baa..9def923963ac 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java @@ -28,7 +28,7 @@ public class NessieContainer { private static final Logger log = Logger.get(NessieContainer.class); - public static final String DEFAULT_IMAGE = "ghcr.io/projectnessie/nessie:0.105.3"; + public static final String DEFAULT_IMAGE = "ghcr.io/projectnessie/nessie:0.105.6"; public static final String DEFAULT_HOST_NAME = "nessie"; public static final String VERSION_STORE_TYPE = "IN_MEMORY"; diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/UnityCatalogContainer.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/UnityCatalogContainer.java index aeaf9c3ab9f9..e4aafae39d21 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/UnityCatalogContainer.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/UnityCatalogContainer.java @@ -26,7 +26,7 @@ import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; @@ -58,7 +58,7 @@ public class UnityCatalogContainer private final String catalogName; private final String schemaName; - private final PostgreSQLContainer postgreSql; + private final PostgreSQLContainer postgreSql; private final GenericContainer unityCatalog; private final QueryRunner queryRunner; private final AutoCloseableCloser closer = AutoCloseableCloser.create(); @@ -72,8 +72,7 @@ public UnityCatalogContainer(String catalogName, String schemaName) Network network = Network.newNetwork(); closer.register(network); - //noinspection resource - postgreSql = new PostgreSQLContainer<>(DockerImageName.parse("postgres")) + postgreSql = new PostgreSQLContainer(DockerImageName.parse("postgres")) .withNetwork(network) .withNetworkAliases("postgres"); postgreSql.start(); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergAddFilesProcedure.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergAddFilesProcedure.java index f4550150a91f..f7903b085acb 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergAddFilesProcedure.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergAddFilesProcedure.java @@ -124,12 +124,12 @@ private void testAddFilesDifferentFileFormat(String hiveFormat, String icebergFo String hiveTableName = "test_add_files_" + randomNameSuffix(); String icebergTableName = "test_add_files_" + randomNameSuffix(); - assertUpdate("CREATE TABLE iceberg.tpch." + icebergTableName + " WITH (format = '" + icebergFormat + "') AS SELECT 1 x", 1); - assertUpdate("CREATE TABLE hive.tpch." + hiveTableName + " WITH (format = '" + hiveFormat + "') AS SELECT 2 x", 1); + assertUpdate("CREATE TABLE iceberg.tpch." + icebergTableName + " WITH (format = '" + icebergFormat + "') AS SELECT 1 x, 2 y", 1); + assertUpdate("CREATE TABLE hive.tpch." + hiveTableName + " WITH (format = '" + hiveFormat + "') AS SELECT 3 x, 4 y", 1); assertUpdate("ALTER TABLE " + icebergTableName + " EXECUTE add_files_from_table('tpch', '" + hiveTableName + "')"); - assertQuery("SELECT * FROM " + icebergTableName, "VALUES 1, 2"); + assertQuery("SELECT * FROM " + icebergTableName, "VALUES (1, 2), (3, 4)"); assertUpdate("DROP TABLE hive.tpch." + hiveTableName); assertUpdate("DROP TABLE iceberg.tpch." + icebergTableName); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergMigrateProcedure.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergMigrateProcedure.java index 3ef35ce4bc36..f265e67daf4a 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergMigrateProcedure.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergMigrateProcedure.java @@ -30,6 +30,8 @@ import static com.google.common.collect.MoreCollectors.onlyElement; import static io.trino.plugin.iceberg.IcebergFileFormat.AVRO; +import static io.trino.plugin.iceberg.IcebergFileFormat.ORC; +import static io.trino.plugin.iceberg.IcebergFileFormat.PARQUET; import static io.trino.testing.TestingNames.randomNameSuffix; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; @@ -289,12 +291,19 @@ public static Object[][] fileFormats() @Test public void testMigratePartitionedTable() + { + testMigratePartitionedTable(PARQUET); + testMigratePartitionedTable(ORC); + testMigratePartitionedTable(AVRO); + } + + private void testMigratePartitionedTable(IcebergFileFormat format) { String tableName = "test_migrate_partitioned_" + randomNameSuffix(); String hiveTableName = "hive.tpch." + tableName; String icebergTableName = "iceberg.tpch." + tableName; - assertUpdate("CREATE TABLE " + hiveTableName + " WITH (partitioned_by = ARRAY['part_col']) AS SELECT 1 id, 'part1' part_col", 1); + assertUpdate("CREATE TABLE " + hiveTableName + " WITH (format = '" + format + "', partitioned_by = ARRAY['part_col']) AS SELECT 1 id, 'part1' part_col", 1); assertQueryFails("SELECT * FROM " + icebergTableName, "Not an Iceberg table: .*"); assertUpdate("CALL iceberg.system.migrate('tpch', '" + tableName + "')"); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergOptimizeManifestsProcedure.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergOptimizeManifestsProcedure.java index bc59e678bfc2..67a41d75f0e9 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergOptimizeManifestsProcedure.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/procedure/TestIcebergOptimizeManifestsProcedure.java @@ -13,22 +13,33 @@ */ package io.trino.plugin.iceberg.procedure; +import com.google.common.collect.ImmutableMap; import io.trino.filesystem.TrinoFileSystemFactory; import io.trino.metastore.HiveMetastore; import io.trino.plugin.iceberg.IcebergQueryRunner; import io.trino.plugin.iceberg.IcebergTestUtils; +import io.trino.plugin.iceberg.catalog.TrinoCatalog; +import io.trino.spi.connector.SchemaTableName; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; import io.trino.testing.QueryRunner; import io.trino.testing.sql.TestTable; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; +import java.util.Optional; import java.util.Set; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.trino.plugin.iceberg.IcebergTestUtils.SESSION; import static io.trino.plugin.iceberg.IcebergTestUtils.getFileSystemFactory; import static io.trino.plugin.iceberg.IcebergTestUtils.getHiveMetastore; +import static io.trino.plugin.iceberg.IcebergTestUtils.getTrinoCatalog; +import static io.trino.testing.TestingNames.randomNameSuffix; import static org.assertj.core.api.Assertions.assertThat; final class TestIcebergOptimizeManifestsProcedure @@ -36,6 +47,7 @@ final class TestIcebergOptimizeManifestsProcedure { private HiveMetastore metastore; private TrinoFileSystemFactory fileSystemFactory; + private TrinoCatalog catalog; @Override protected QueryRunner createQueryRunner() @@ -44,6 +56,7 @@ protected QueryRunner createQueryRunner() DistributedQueryRunner queryRunner = IcebergQueryRunner.builder().build(); metastore = getHiveMetastore(queryRunner); fileSystemFactory = getFileSystemFactory(queryRunner); + catalog = getTrinoCatalog(metastore, fileSystemFactory, "iceberg"); return queryRunner; } @@ -217,6 +230,25 @@ void testNotRewriteSingleManifest() } } + @Test + void testNoSnapshot() + { + SchemaTableName tableName = new SchemaTableName("tpch", "test_no_snapshot" + randomNameSuffix()); + + catalog.newCreateTableTransaction( + SESSION, + tableName, + new Schema(Types.NestedField.required(1, "x", Types.LongType.get())), + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + Optional.ofNullable(catalog.defaultTableLocation(SESSION, tableName)), + ImmutableMap.of()) + .commitTransaction(); + assertThat(catalog.loadTable(SESSION, tableName).currentSnapshot()).isNull(); + + assertUpdate("ALTER TABLE " + tableName + " EXECUTE optimize_manifests"); + } + @Test void testUnsupportedWhere() { diff --git a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java index 5472c9422280..83e492561b7f 100644 --- a/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java +++ b/plugin/trino-iceberg/src/test/java/org/apache/iceberg/snowflake/TestTrinoSnowflakeCatalog.java @@ -21,7 +21,6 @@ import io.trino.filesystem.s3.S3FileSystemFactory; import io.trino.filesystem.s3.S3FileSystemStats; import io.trino.metastore.TableInfo; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.iceberg.ColumnIdentity; import io.trino.plugin.iceberg.CommitTaskData; import io.trino.plugin.iceberg.IcebergMetadata; @@ -33,6 +32,7 @@ import io.trino.plugin.iceberg.catalog.snowflake.SnowflakeIcebergTableOperationsProvider; import io.trino.plugin.iceberg.catalog.snowflake.TestingSnowflakeServer; import io.trino.plugin.iceberg.catalog.snowflake.TrinoSnowflakeCatalog; +import io.trino.spi.NodeVersion; import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorViewDefinition; @@ -48,6 +48,7 @@ import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.jdbc.JdbcClientPool; import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -148,6 +149,12 @@ private static void executeOnSnowflake(TestingSnowflakeServer server, String sql server.execute(SNOWFLAKE_TEST_SCHEMA, sql); } + @Override + protected void createNamespaceWithProperties(TrinoCatalog catalog, String namespace, Map namespaceProperties) + { + Assumptions.abort("Snowflake catalog does not support creating namespaces"); + } + @Override protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations) { diff --git a/plugin/trino-ignite/pom.xml b/plugin/trino-ignite/pom.xml index 31d459ffe170..38fb05dff3e5 100644 --- a/plugin/trino-ignite/pom.xml +++ b/plugin/trino-ignite/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -229,13 +229,13 @@ org.testcontainers - jdbc + testcontainers test org.testcontainers - testcontainers + testcontainers-jdbc test diff --git a/plugin/trino-jmx/pom.xml b/plugin/trino-jmx/pom.xml index b51edf65485e..f63c76441712 100644 --- a/plugin/trino-jmx/pom.xml +++ b/plugin/trino-jmx/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-jmx/src/main/java/io/trino/plugin/jmx/JmxConnectorFactory.java b/plugin/trino-jmx/src/main/java/io/trino/plugin/jmx/JmxConnectorFactory.java index ce8cb497bf37..aecfb6e4d2ae 100644 --- a/plugin/trino-jmx/src/main/java/io/trino/plugin/jmx/JmxConnectorFactory.java +++ b/plugin/trino-jmx/src/main/java/io/trino/plugin/jmx/JmxConnectorFactory.java @@ -16,9 +16,8 @@ import com.google.inject.Injector; import com.google.inject.Scopes; import io.airlift.bootstrap.Bootstrap; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; @@ -49,10 +48,9 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, new MBeanServerModule(), + new ConnectorContextModule(catalogName, context), binder -> { configBinder(binder).bindConfig(JmxConnectorConfig.class); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); binder.bind(JmxConnector.class).in(Scopes.SINGLETON); binder.bind(JmxHistoricalData.class).in(Scopes.SINGLETON); binder.bind(JmxMetadata.class).in(Scopes.SINGLETON); @@ -65,6 +63,7 @@ public Connector create(String catalogName, Map config, Connecto Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-jmx/src/test/java/io/trino/plugin/jmx/TestJmxSplitManager.java b/plugin/trino-jmx/src/test/java/io/trino/plugin/jmx/TestJmxSplitManager.java index f3cfb1d13e35..a21fd13094fa 100644 --- a/plugin/trino-jmx/src/test/java/io/trino/plugin/jmx/TestJmxSplitManager.java +++ b/plugin/trino-jmx/src/test/java/io/trino/plugin/jmx/TestJmxSplitManager.java @@ -23,7 +23,6 @@ import io.trino.spi.Node; import io.trino.spi.NodeManager; import io.trino.spi.connector.ColumnHandle; -import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorSplit; import io.trino.spi.connector.ConnectorSplitSource; import io.trino.spi.connector.ConnectorTransactionHandle; @@ -33,6 +32,7 @@ import io.trino.spi.connector.SchemaTableName; import io.trino.spi.predicate.NullableValue; import io.trino.spi.predicate.TupleDomain; +import io.trino.testing.TestingConnectorContext; import io.trino.testing.TestingNodeManager; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; @@ -85,14 +85,7 @@ public class TestJmxSplitManager "jmx.dump-period", format("%dms", JMX_STATS_DUMP.toMillis()), "jmx.max-entries", "1000", "bootstrap.quiet", "true"), - new ConnectorContext() - { - @Override - public NodeManager getNodeManager() - { - return nodeManager; - } - }); + new TestingConnectorContext(nodeManager)); private final JmxColumnHandle columnHandle = new JmxColumnHandle("node", createUnboundedVarcharType()); diff --git a/plugin/trino-kafka-event-listener/pom.xml b/plugin/trino-kafka-event-listener/pom.xml index 5f1c25849fcb..66c3dc923db8 100644 --- a/plugin/trino-kafka-event-listener/pom.xml +++ b/plugin/trino-kafka-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kafka-event-listener/src/main/java/io/trino/plugin/eventlistener/kafka/KafkaEventListenerFactory.java b/plugin/trino-kafka-event-listener/src/main/java/io/trino/plugin/eventlistener/kafka/KafkaEventListenerFactory.java index f90b2238bc0a..22cd56772962 100644 --- a/plugin/trino-kafka-event-listener/src/main/java/io/trino/plugin/eventlistener/kafka/KafkaEventListenerFactory.java +++ b/plugin/trino-kafka-event-listener/src/main/java/io/trino/plugin/eventlistener/kafka/KafkaEventListenerFactory.java @@ -54,6 +54,7 @@ public EventListener create(Map config, EventListenerContext con Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-kafka-event-listener/src/test/java/io/trino/plugin/eventlistener/kafka/TestUtils.java b/plugin/trino-kafka-event-listener/src/test/java/io/trino/plugin/eventlistener/kafka/TestUtils.java index 7ecbae75d8fc..c3aaf9bf6636 100644 --- a/plugin/trino-kafka-event-listener/src/test/java/io/trino/plugin/eventlistener/kafka/TestUtils.java +++ b/plugin/trino-kafka-event-listener/src/test/java/io/trino/plugin/eventlistener/kafka/TestUtils.java @@ -14,6 +14,7 @@ package io.trino.plugin.eventlistener.kafka; +import com.google.common.collect.ImmutableMap; import io.trino.operator.RetryPolicy; import io.trino.spi.ErrorCode; import io.trino.spi.ErrorType; @@ -118,6 +119,7 @@ private TestUtils() {} Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), 0L, 0L, 0L, @@ -144,6 +146,7 @@ private TestUtils() {} Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + ImmutableMap.of(), Optional.empty()); queryFailureInfo = Optional.of( diff --git a/plugin/trino-kafka/pom.xml b/plugin/trino-kafka/pom.xml index 01c4b8d62b47..8a5f41f960a7 100644 --- a/plugin/trino-kafka/pom.xml +++ b/plugin/trino-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java index 290c4316a78e..ddc735e0b86e 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorFactory.java @@ -18,13 +18,12 @@ import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.trino.plugin.base.CatalogNameModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.plugin.kafka.security.KafkaSecurityModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; import java.util.List; import java.util.Map; @@ -58,21 +57,21 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, ImmutableList.builder() - .add(new CatalogNameModule(catalogName)) .add(new JsonModule()) - .add(new TypeDeserializerModule(context.getTypeManager())) - .add(new KafkaConnectorModule(context.getTypeManager())) + .add(new TypeDeserializerModule()) + .add(new KafkaConnectorModule()) .add(new KafkaClientsModule()) .add(new KafkaSecurityModule()) + .add(new ConnectorContextModule(catalogName, context)) .add(binder -> { binder.bind(ClassLoader.class).toInstance(KafkaConnectorFactory.class.getClassLoader()); - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); }) .addAll(extensions) .build()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorModule.java index 57961b02c91f..680b5716c703 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaConnectorModule.java @@ -31,24 +31,15 @@ import io.trino.spi.connector.ConnectorPageSinkProvider; import io.trino.spi.connector.ConnectorRecordSetProvider; import io.trino.spi.connector.ConnectorSplitManager; -import io.trino.spi.type.TypeManager; import static com.google.inject.multibindings.Multibinder.newSetBinder; import static io.airlift.configuration.ConditionalModule.conditionalModule; import static io.airlift.configuration.ConfigBinder.configBinder; import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; -import static java.util.Objects.requireNonNull; public class KafkaConnectorModule extends AbstractConfigurationAwareModule { - private final TypeManager typeManager; - - public KafkaConnectorModule(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - @Override public void setup(Binder binder) { @@ -67,7 +58,7 @@ public void setup(Binder binder) configBinder(binder).bindConfig(KafkaConfig.class); bindTopicSchemaProviderModule(FileTableDescriptionSupplier.NAME, new FileTableDescriptionSupplierModule()); - bindTopicSchemaProviderModule(ConfluentSchemaRegistryTableDescriptionSupplier.NAME, new ConfluentModule(typeManager)); + bindTopicSchemaProviderModule(ConfluentSchemaRegistryTableDescriptionSupplier.NAME, new ConfluentModule()); newSetBinder(binder, SessionPropertiesProvider.class).addBinding().to(KafkaSessionProperties.class).in(Scopes.SINGLETON); jsonCodecBinder(binder).bindJsonCodec(KafkaTopicDescription.class); } diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaFilterManager.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaFilterManager.java index 5c853d9011ac..25d3c83301cd 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaFilterManager.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/KafkaFilterManager.java @@ -182,8 +182,7 @@ private static Map> findOffsetsForTimestampGreate { Map topicPartitionOffsetAndTimestamps = kafkaConsumer.offsetsForTimes(timestamps); return topicPartitionOffsetAndTimestamps.entrySet().stream() - .collect(toMap(Map.Entry::getKey, entry -> Optional.of(entry.getValue()) - .map(OffsetAndTimestamp::offset))); + .collect(toMap(Map.Entry::getKey, entry -> Optional.ofNullable(entry.getValue()).map(OffsetAndTimestamp::offset))); } private static Map overridePartitionBeginOffsets(Map partitionBeginOffsets, diff --git a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/schema/confluent/ConfluentModule.java b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/schema/confluent/ConfluentModule.java index ec980621a82e..ffd321c65210 100644 --- a/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/schema/confluent/ConfluentModule.java +++ b/plugin/trino-kafka/src/main/java/io/trino/plugin/kafka/schema/confluent/ConfluentModule.java @@ -79,18 +79,9 @@ public class ConfluentModule extends AbstractConfigurationAwareModule { - private final TypeManager typeManager; - - public ConfluentModule(TypeManager typeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - } - @Override protected void setup(Binder binder) { - binder.bind(TypeManager.class).toInstance(typeManager); - configBinder(binder).bindConfig(ConfluentSchemaRegistryConfig.class); install(new ConfluentDecoderModule()); install(new ConfluentEncoderModule()); diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConnectorTest.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConnectorTest.java index 8fb0f718956f..1e85b21bca8c 100644 --- a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConnectorTest.java +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/TestKafkaConnectorTest.java @@ -170,6 +170,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_VIEW, SUPPORTS_DELETE, SUPPORTS_DEREFERENCE_PUSHDOWN, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MERGE, SUPPORTS_RENAME_COLUMN, SUPPORTS_RENAME_TABLE, diff --git a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/protobuf/TestProtobufEncoder.java b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/protobuf/TestProtobufEncoder.java index da9b8bae14ca..bb3405bbba1a 100644 --- a/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/protobuf/TestProtobufEncoder.java +++ b/plugin/trino-kafka/src/test/java/io/trino/plugin/kafka/protobuf/TestProtobufEncoder.java @@ -399,10 +399,7 @@ private void testNestedStructuralDataTypes(String stringData, Integer integerDat new int[] {0, 1}, nativeValueToBlock(VARCHAR, utf8Slice("Key")), rowBlockBuilder.build()); - mapType.appendTo( - mapBlock, - 0, - mapBlockBuilder); + mapBlockBuilder.append(mapBlock.getUnderlyingValueBlock(), mapBlock.getUnderlyingValuePosition(0)); Type listType = nestedRowType.getTypeParameters().get(0); BlockBuilder listBlockBuilder = listType.createBlockBuilder(null, 1); @@ -411,11 +408,11 @@ private void testNestedStructuralDataTypes(String stringData, Integer integerDat Optional.empty(), new int[] {0, rowBlockBuilder.getPositionCount()}, rowBlockBuilder.build()); - listType.appendTo(arrayBlock, 0, listBlockBuilder); + listBlockBuilder.append(arrayBlock.getUnderlyingValueBlock(), arrayBlock.getUnderlyingValuePosition(0)); BlockBuilder nestedBlockBuilder = nestedRowType.createBlockBuilder(null, 1); Block rowBlock = fromFieldBlocks(1, new Block[] {listBlockBuilder.build(), mapBlockBuilder.build(), rowBlockBuilder.build()}); - nestedRowType.appendTo(rowBlock, 0, nestedBlockBuilder); + nestedBlockBuilder.append(rowBlock.getUnderlyingValueBlock(), rowBlock.getUnderlyingValuePosition(0)); rowEncoder.appendColumnValue(nestedBlockBuilder.build(), 0); diff --git a/plugin/trino-lakehouse/pom.xml b/plugin/trino-lakehouse/pom.xml index b32a4094d048..67148a1a1754 100644 --- a/plugin/trino-lakehouse/pom.xml +++ b/plugin/trino-lakehouse/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -128,6 +128,12 @@ provided + + io.airlift + log + runtime + + io.airlift log-manager @@ -227,6 +233,12 @@ test + + io.trino.tpch + tpch + test + + org.assertj assertj-core @@ -248,6 +260,20 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + org.basepom.maven duplicate-finder-maven-plugin diff --git a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseConnectorFactory.java b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseConnectorFactory.java index b0fc676c3e62..e58f61511ed1 100644 --- a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseConnectorFactory.java +++ b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseConnectorFactory.java @@ -16,24 +16,15 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.trace.Tracer; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.plugin.hive.NodeVersion; import io.trino.plugin.hive.security.HiveSecurityModule; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.PageIndexerFactory; -import io.trino.spi.PageSorter; -import io.trino.spi.VersionEmbedder; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.classloader.ThreadContextClassLoader; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.connector.MetadataProvider; import org.weakref.jmx.guice.MBeanModule; import java.util.Map; @@ -60,29 +51,19 @@ public Connector create(String catalogName, Map config, Connecto new MBeanServerModule(), new ConnectorObjectNameGeneratorModule("io.trino.plugin", "trino.plugin"), new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), new LakehouseModule(), new LakehouseHiveModule(), new LakehouseIcebergModule(), new LakehouseDeltaModule(), new LakehouseHudiModule(), new HiveSecurityModule(), - new LakehouseFileSystemModule(catalogName, context.getCurrentNode().isCoordinator(), context.getOpenTelemetry()), - binder -> { - binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()); - binder.bind(Tracer.class).toInstance(context.getTracer()); - binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getCurrentNode().getVersion())); - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(NodeManager.class).toInstance(context.getNodeManager()); - binder.bind(VersionEmbedder.class).toInstance(context.getVersionEmbedder()); - binder.bind(MetadataProvider.class).toInstance(context.getMetadataProvider()); - binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - binder.bind(PageSorter.class).toInstance(context.getPageSorter()); - }); + new LakehouseFileSystemModule(catalogName, context), + new ConnectorContextModule(catalogName, context)); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseFileSystemModule.java b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseFileSystemModule.java index b480c4311d96..cbf15c2b6a44 100644 --- a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseFileSystemModule.java +++ b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseFileSystemModule.java @@ -15,9 +15,9 @@ import com.google.inject.Binder; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.opentelemetry.api.OpenTelemetry; import io.trino.filesystem.manager.FileSystemModule; import io.trino.plugin.iceberg.IcebergConfig; +import io.trino.spi.connector.ConnectorContext; import static java.util.Objects.requireNonNull; @@ -25,20 +25,18 @@ class LakehouseFileSystemModule extends AbstractConfigurationAwareModule { private final String catalogName; - private final boolean isCoordinator; - private final OpenTelemetry openTelemetry; + private final ConnectorContext context; - public LakehouseFileSystemModule(String catalogName, boolean isCoordinator, OpenTelemetry openTelemetry) + public LakehouseFileSystemModule(String catalogName, ConnectorContext context) { this.catalogName = requireNonNull(catalogName, "catalogName is null"); - this.isCoordinator = isCoordinator; - this.openTelemetry = openTelemetry; + this.context = requireNonNull(context, "context is null"); } @Override protected void setup(Binder binder) { boolean metadataCacheEnabled = buildConfigObject(IcebergConfig.class).isMetadataCacheEnabled(); - install(new FileSystemModule(catalogName, isCoordinator, openTelemetry, metadataCacheEnabled)); + install(new FileSystemModule(catalogName, context, metadataCacheEnabled)); } } diff --git a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseHiveModule.java b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseHiveModule.java index 2892a81d0116..eaa6c3f0223e 100644 --- a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseHiveModule.java +++ b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseHiveModule.java @@ -40,6 +40,7 @@ import io.trino.plugin.hive.TransactionalMetadataFactory; import io.trino.plugin.hive.avro.AvroFileWriterFactory; import io.trino.plugin.hive.avro.AvroPageSourceFactory; +import io.trino.plugin.hive.crypto.ParquetEncryptionModule; import io.trino.plugin.hive.fs.CachingDirectoryLister; import io.trino.plugin.hive.fs.DirectoryLister; import io.trino.plugin.hive.fs.TransactionScopeCachingDirectoryListerFactory; @@ -76,7 +77,7 @@ class LakehouseHiveModule @Override protected void setup(Binder binder) { - install(new HiveMetastoreModule(Optional.empty())); + install(new HiveMetastoreModule(Optional.empty(), false)); configBinder(binder).bindConfig(HiveConfig.class); configBinder(binder).bindConfig(HiveMetastoreConfig.class); @@ -136,5 +137,6 @@ protected void setup(Binder binder) fileWriterFactoryBinder.addBinding().to(ParquetFileWriterFactory.class).in(Scopes.SINGLETON); binder.install(new HiveExecutorModule()); + install(new ParquetEncryptionModule()); } } diff --git a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseMetadata.java b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseMetadata.java index fd581d527d51..b70c7af086aa 100644 --- a/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseMetadata.java +++ b/plugin/trino-lakehouse/src/main/java/io/trino/plugin/lakehouse/LakehouseMetadata.java @@ -13,6 +13,7 @@ */ package io.trino.plugin.lakehouse; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; import io.airlift.slice.Slice; import io.trino.metastore.Table; @@ -87,6 +88,8 @@ import io.trino.spi.expression.Constant; import io.trino.spi.function.LanguageFunction; import io.trino.spi.function.SchemaFunctionName; +import io.trino.spi.metrics.Metric; +import io.trino.spi.metrics.Metrics; import io.trino.spi.security.GrantInfo; import io.trino.spi.security.Privilege; import io.trino.spi.security.RoleGrant; @@ -198,9 +201,9 @@ public void finishTableExecute(ConnectorSession session, ConnectorTableExecuteHa } @Override - public void executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) + public Map executeTableExecute(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle) { - forHandle(tableExecuteHandle).executeTableExecute(session, tableExecuteHandle); + return forHandle(tableExecuteHandle).executeTableExecute(session, tableExecuteHandle); } @Override @@ -251,6 +254,25 @@ public Optional getInfo(ConnectorSession session, ConnectorTableHandle t return forHandle(table).getInfo(session, table); } + @Override + public Metrics getMetrics(ConnectorSession session) + { + ImmutableMap.Builder> metrics = ImmutableMap.>builder(); + hiveMetadata.getMetrics(session).getMetrics().forEach((metricName, metric) -> { + metrics.put("hive." + metricName, metric); + }); + icebergMetadata.getMetrics(session).getMetrics().forEach((metricName, metric) -> { + metrics.put("iceberg." + metricName, metric); + }); + deltaMetadata.getMetrics(session).getMetrics().forEach((metricName, metric) -> { + metrics.put("delta." + metricName, metric); + }); + hudiMetadata.getMetrics(session).getMetrics().forEach((metricName, metric) -> { + metrics.put("hudi." + metricName, metric); + }); + return new Metrics(metrics.buildOrThrow()); + } + @Override public List listTables(ConnectorSession session, Optional schemaName) { @@ -476,10 +498,16 @@ public Optional getInsertLayout(ConnectorSession session, return forHandle(tableHandle).getInsertLayout(session, tableHandle); } + @Override + public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean tableReplace) + { + return forProperties(tableMetadata.getProperties()).getStatisticsCollectionMetadataForWrite(session, unwrapTableMetadata(tableMetadata), tableReplace); + } + @Override public TableStatisticsMetadata getStatisticsCollectionMetadataForWrite(ConnectorSession session, ConnectorTableMetadata tableMetadata) { - return forProperties(tableMetadata.getProperties()).getStatisticsCollectionMetadataForWrite(session, unwrapTableMetadata(tableMetadata)); + throw new UnsupportedOperationException("This variant of getStatisticsCollectionMetadataForWrite is unsupported"); } @Override diff --git a/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/LakehouseQueryRunner.java b/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/LakehouseQueryRunner.java index 8d2bda098bfa..55491276e5d5 100644 --- a/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/LakehouseQueryRunner.java +++ b/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/LakehouseQueryRunner.java @@ -14,19 +14,32 @@ package io.trino.plugin.lakehouse; import com.google.common.collect.ImmutableMap; +import io.airlift.log.Level; +import io.airlift.log.Logger; import io.airlift.log.Logging; import io.trino.plugin.iceberg.IcebergPlugin; import io.trino.plugin.tpcds.TpcdsPlugin; import io.trino.plugin.tpch.TpchPlugin; import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import io.trino.tpch.TpchTable; + +import java.io.File; +import java.util.Set; import static io.airlift.testing.Closeables.closeAllSuppress; +import static io.trino.plugin.lakehouse.TableType.DELTA; +import static io.trino.plugin.lakehouse.TableType.HIVE; +import static io.trino.plugin.lakehouse.TableType.ICEBERG; import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.nio.file.Files.createTempDirectory; +import static java.util.Locale.ENGLISH; public final class LakehouseQueryRunner { static { - Logging.initialize(); + Logging logging = Logging.initialize(); + logging.setLevel("org.apache.iceberg", Level.OFF); } private LakehouseQueryRunner() {} @@ -81,4 +94,32 @@ public DistributedQueryRunner build() } } } + + public static void main(String[] args) + throws Exception + { + File metastoreDir = createTempDirectory("delta_query_runner").toFile(); + metastoreDir.deleteOnExit(); + + @SuppressWarnings("resource") + QueryRunner queryRunner = builder() + .addCoordinatorProperty("http-server.http.port", "8080") + .addLakehouseProperty("hive.metastore", "file") + .addLakehouseProperty("hive.metastore.catalog.dir", metastoreDir.toURI().toString()) + .addLakehouseProperty("fs.hadoop.enabled", "true") + .build(); + + for (TableType tableType : Set.of(HIVE, ICEBERG, DELTA)) { + String type = tableType.name().toLowerCase(ENGLISH); + queryRunner.execute("CREATE SCHEMA " + type); + for (TpchTable table : TpchTable.getTables()) { + queryRunner.execute("CREATE TABLE " + type + "." + table.getTableName() + " WITH (type = '" + type + "')" + + "AS SELECT * FROM tpch.tiny." + table.getTableName()); + } + } + + Logger log = Logger.get(LakehouseQueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } } diff --git a/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/TestLakehouseConnectorTest.java b/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/TestLakehouseConnectorTest.java index 0e4bc7787ff4..465790ae611b 100644 --- a/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/TestLakehouseConnectorTest.java +++ b/plugin/trino-lakehouse/src/test/java/io/trino/plugin/lakehouse/TestLakehouseConnectorTest.java @@ -13,21 +13,30 @@ */ package io.trino.plugin.lakehouse; +import com.google.common.collect.ImmutableList; import io.trino.Session; +import io.trino.connector.MockConnectorFactory; +import io.trino.connector.MockConnectorPlugin; import io.trino.plugin.hive.containers.Hive3MinioDataLake; +import io.trino.spi.metrics.Metrics; +import io.trino.spi.statistics.TableStatistics; import io.trino.testing.BaseConnectorTest; import io.trino.testing.MaterializedResult; +import io.trino.testing.QueryFailedException; import io.trino.testing.QueryRunner; import io.trino.testing.TestingConnectorBehavior; +import io.trino.testing.TestingSession; import io.trino.testing.sql.TestTable; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import static io.trino.SystemSessionProperties.ITERATIVE_OPTIMIZER_TIMEOUT; import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static io.trino.spi.type.VarcharType.VARCHAR; import static io.trino.testing.MaterializedResult.resultBuilder; @@ -36,8 +45,10 @@ import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY; import static io.trino.testing.containers.Minio.MINIO_REGION; import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY; +import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Fail.fail; import static org.junit.jupiter.api.Assumptions.abort; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -81,6 +92,29 @@ public void setUp() copyTpchTables(getQueryRunner(), "tpch", TINY_SCHEMA_NAME, REQUIRED_TPCH_TABLES); } + @BeforeAll + public void initMockMetricsCatalog() + { + QueryRunner queryRunner = getQueryRunner(); + String mockConnector = "mock_metrics"; + queryRunner.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder() + .withName(mockConnector) + .withListSchemaNames(_ -> ImmutableList.of("default")) + .withGetTableStatistics(_ -> { + try { + Thread.sleep(110); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + return TableStatistics.empty(); + }) + .build())); + + queryRunner.createCatalog("mock_metrics", mockConnector); + } + @Override protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) { @@ -90,6 +124,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_REPORTING_WRITTEN_BYTES -> true; case SUPPORTS_ADD_COLUMN_NOT_NULL_CONSTRAINT, SUPPORTS_DEFAULT_COLUMN_VALUE, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_REFRESH_VIEW, SUPPORTS_RENAME_MATERIALIZED_VIEW_ACROSS_SCHEMAS, SUPPORTS_TOPN_PUSHDOWN -> false; @@ -365,4 +400,36 @@ public void testShowCreateTable() type = 'ICEBERG' )\\E"""); } + + @Test + public void testCatalogMetadataMetrics() + { + QueryRunner.MaterializedResultWithPlan result = getQueryRunner().executeWithPlan( + getSession(), + "SELECT count(*) FROM region r, nation n WHERE r.regionkey = n.regionkey"); + Map metrics = getCatalogMetadataMetrics(result.queryId()); + assertCountMetricExists(metrics, "lakehouse", "iceberg.metastore.all.time.total"); + assertDistributionMetricExists(metrics, "lakehouse", "iceberg.metastore.all.time.distribution"); + assertCountMetricExists(metrics, "lakehouse", "iceberg.metastore.getTable.time.total"); + assertDistributionMetricExists(metrics, "lakehouse", "iceberg.metastore.getTable.time.distribution"); + } + + @Test + public void testCatalogMetadataMetricsWithOptimizerTimeoutExceeded() + { + String query = "SELECT count(*) FROM region r, nation n, mock_metrics.default.mock_table m WHERE r.regionkey = n.regionkey"; + try { + Session smallOptimizerTimeout = TestingSession.testSessionBuilder(getSession()) + .setSystemProperty(ITERATIVE_OPTIMIZER_TIMEOUT, "100ms") + .build(); + QueryRunner.MaterializedResultWithPlan result = getQueryRunner().executeWithPlan(smallOptimizerTimeout, query); + fail(format("Expected query to fail: %s [QueryId: %s]", query, result.queryId())); + } + catch (QueryFailedException e) { + assertThat(e.getMessage()).contains("The optimizer exhausted the time limit"); + Map metrics = getCatalogMetadataMetrics(e.getQueryId()); + assertCountMetricExists(metrics, "lakehouse", "iceberg.metastore.all.time.total"); + assertCountMetricExists(metrics, "lakehouse", "iceberg.metastore.getTable.time.total"); + } + } } diff --git a/plugin/trino-ldap-group-provider/pom.xml b/plugin/trino-ldap-group-provider/pom.xml index 5b062c4b4194..fa21edbb62ba 100644 --- a/plugin/trino-ldap-group-provider/pom.xml +++ b/plugin/trino-ldap-group-provider/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -142,13 +142,13 @@ org.testcontainers - junit-jupiter + testcontainers test org.testcontainers - testcontainers + testcontainers-junit-jupiter test diff --git a/plugin/trino-ldap-group-provider/src/main/java/io/trino/plugin/ldapgroup/LdapGroupProviderFactory.java b/plugin/trino-ldap-group-provider/src/main/java/io/trino/plugin/ldapgroup/LdapGroupProviderFactory.java index a000da989cb5..d2cd0452bd21 100644 --- a/plugin/trino-ldap-group-provider/src/main/java/io/trino/plugin/ldapgroup/LdapGroupProviderFactory.java +++ b/plugin/trino-ldap-group-provider/src/main/java/io/trino/plugin/ldapgroup/LdapGroupProviderFactory.java @@ -44,6 +44,7 @@ public GroupProvider create(Map requiredConfig) Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(requiredConfig) .initialize(); diff --git a/plugin/trino-loki/pom.xml b/plugin/trino-loki/pom.xml index 68e5ef1a858a..f8bca529e8b3 100644 --- a/plugin/trino-loki/pom.xml +++ b/plugin/trino-loki/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -53,7 +53,6 @@ io.github.jeschkies loki-client - 0.0.5 @@ -108,6 +107,12 @@ runtime + + com.squareup.okhttp3 + okhttp-jvm + runtime + + io.airlift log diff --git a/plugin/trino-loki/src/main/java/io/trino/plugin/loki/LokiConnectorFactory.java b/plugin/trino-loki/src/main/java/io/trino/plugin/loki/LokiConnectorFactory.java index 65328a27e045..dc6aef83015f 100644 --- a/plugin/trino-loki/src/main/java/io/trino/plugin/loki/LokiConnectorFactory.java +++ b/plugin/trino-loki/src/main/java/io/trino/plugin/loki/LokiConnectorFactory.java @@ -16,6 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -47,11 +48,13 @@ public Connector create(String catalogName, Map requiredConfig, Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), + new ConnectorContextModule(catalogName, context), new LokiModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(requiredConfig) .initialize(); diff --git a/plugin/trino-mariadb/pom.xml b/plugin/trino-mariadb/pom.xml index c2312e268757..b488189d3f79 100644 --- a/plugin/trino-mariadb/pom.xml +++ b/plugin/trino-mariadb/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -210,13 +210,13 @@ org.testcontainers - mariadb + testcontainers test org.testcontainers - testcontainers + testcontainers-mariadb test diff --git a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestingMariaDbServer.java b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestingMariaDbServer.java index 45aadf2bb260..bc0b61017fe1 100644 --- a/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestingMariaDbServer.java +++ b/plugin/trino-mariadb/src/test/java/io/trino/plugin/mariadb/TestingMariaDbServer.java @@ -13,7 +13,7 @@ */ package io.trino.plugin.mariadb; -import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.mariadb.MariaDBContainer; import org.testcontainers.utility.DockerImageName; import java.sql.Connection; @@ -30,7 +30,7 @@ public class TestingMariaDbServer public static final String DEFAULT_VERSION = "10.10"; private static final int MARIADB_PORT = 3306; - private final MariaDBContainer container; + private final MariaDBContainer container; public TestingMariaDbServer() { @@ -39,7 +39,7 @@ public TestingMariaDbServer() public TestingMariaDbServer(String tag) { - container = new MariaDBContainer<>(DockerImageName.parse("mariadb").withTag(tag)) + container = new MariaDBContainer(DockerImageName.parse("mariadb").withTag(tag)) .withDatabaseName("tpch"); // character-set-server：the default character set is latin1 // explicit-defaults-for-timestamp: 1 is ON, the default set is 0 (OFF) diff --git a/plugin/trino-memory/pom.xml b/plugin/trino-memory/pom.xml index 9858d7d184e0..ba4acd664646 100644 --- a/plugin/trino-memory/pom.xml +++ b/plugin/trino-memory/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryConnectorFactory.java b/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryConnectorFactory.java index cc68998689eb..d40fddfd4a49 100644 --- a/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryConnectorFactory.java +++ b/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryConnectorFactory.java @@ -16,6 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; @@ -44,10 +45,12 @@ public Connector create(String catalogName, Map requiredConfig, Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, new JsonModule(), - new MemoryModule(context.getTypeManager(), context.getCurrentNode(), context.getNodeManager())); + new ConnectorContextModule(catalogName, context), + new MemoryModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(requiredConfig) .initialize(); diff --git a/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryModule.java b/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryModule.java index d26ca1f9547a..89be8e66cb76 100644 --- a/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryModule.java +++ b/plugin/trino-memory/src/main/java/io/trino/plugin/memory/MemoryModule.java @@ -16,34 +16,15 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; -import io.trino.spi.Node; -import io.trino.spi.NodeManager; -import io.trino.spi.type.TypeManager; import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.util.Objects.requireNonNull; public class MemoryModule implements Module { - private final TypeManager typeManager; - private final Node currentNode; - private final NodeManager nodeManager; - - public MemoryModule(TypeManager typeManager, Node currentNode, NodeManager nodeManager) - { - this.typeManager = requireNonNull(typeManager, "typeManager is null"); - this.currentNode = requireNonNull(currentNode, "currentNode is null"); - this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); - } - @Override public void configure(Binder binder) { - binder.bind(TypeManager.class).toInstance(typeManager); - binder.bind(Node.class).toInstance(currentNode); - binder.bind(NodeManager.class).toInstance(nodeManager); - binder.bind(MemoryConnector.class).in(Scopes.SINGLETON); binder.bind(MemoryMetadata.class).in(Scopes.SINGLETON); binder.bind(MemorySplitManager.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java index 048f3a8b5db6..98fffc30f6fc 100644 --- a/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java +++ b/plugin/trino-memory/src/test/java/io/trino/plugin/memory/TestMemoryConnectorTest.java @@ -96,7 +96,6 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_DELETE, SUPPORTS_DEREFERENCE_PUSHDOWN, SUPPORTS_DROP_COLUMN, - SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MERGE, SUPPORTS_PREDICATE_PUSHDOWN, SUPPORTS_RENAME_FIELD, diff --git a/plugin/trino-ml/pom.xml b/plugin/trino-ml/pom.xml index c4f8c4a4bb69..a4009586a001 100644 --- a/plugin/trino-ml/pom.xml +++ b/plugin/trino-ml/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-mongodb/pom.xml b/plugin/trino-mongodb/pom.xml index 8710fe628384..3093ade1628f 100644 --- a/plugin/trino-mongodb/pom.xml +++ b/plugin/trino-mongodb/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -15,7 +15,7 @@ true - 5.6.0 + 5.6.1 @@ -265,13 +265,13 @@ org.testcontainers - mongodb + testcontainers test org.testcontainers - testcontainers + testcontainers-mongodb test diff --git a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoClientModule.java b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoClientModule.java index f4836c083b5b..ecfae4f649aa 100644 --- a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoClientModule.java +++ b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoClientModule.java @@ -13,6 +13,8 @@ */ package io.trino.plugin.mongodb; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.inject.Binder; import com.google.inject.Provides; @@ -76,10 +78,12 @@ public void setup(Binder binder) @Provides public static MongoSession createMongoSession(TypeManager typeManager, MongoClientConfig config, Set configurators, OpenTelemetry openTelemetry) { - MongoClientSettings.Builder options = MongoClientSettings.builder(); - configurators.forEach(configurator -> configurator.configure(options)); - options.addCommandListener(MongoTelemetry.builder(openTelemetry).build().newCommandListener()); - MongoClient client = MongoClients.create(options.build()); + Supplier client = Suppliers.memoize(() -> { + MongoClientSettings.Builder options = MongoClientSettings.builder(); + configurators.forEach(configurator -> configurator.configure(options)); + options.addCommandListener(MongoTelemetry.builder(openTelemetry).build().newCommandListener()); + return MongoClients.create(options.build()); + }); return new MongoSession( typeManager, diff --git a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoConnectorFactory.java b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoConnectorFactory.java index c5e62c42f8c7..f5babefaa252 100644 --- a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoConnectorFactory.java +++ b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoConnectorFactory.java @@ -16,11 +16,10 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; import java.util.Map; @@ -56,10 +55,11 @@ public Connector create(String catalogName, Map config, Connecto "io.trino.bootstrap.catalog." + catalogName, new JsonModule(), new MongoClientModule(), - binder -> binder.bind(TypeManager.class).toInstance(context.getTypeManager()), - binder -> binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry())); + new ConnectorContextModule(catalogName, context)); - Injector injector = app.doNotInitializeLogging() + Injector injector = app + .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoSession.java b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoSession.java index a1cac131cb6e..a5ccbe0d3581 100644 --- a/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoSession.java +++ b/plugin/trino-mongodb/src/main/java/io/trino/plugin/mongodb/MongoSession.java @@ -14,6 +14,7 @@ package io.trino.plugin.mongodb; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Supplier; import com.google.common.cache.Cache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -174,7 +175,7 @@ public class MongoSession .from(Comparator.comparingInt(columnHandle -> columnHandle.dereferenceNames().size())); private final TypeManager typeManager; - private final MongoClient client; + private final Supplier client; private final String schemaCollection; private final boolean caseInsensitiveNameMatching; @@ -183,7 +184,7 @@ public class MongoSession private final Cache tableCache; private final String implicitPrefix; - public MongoSession(TypeManager typeManager, MongoClient client, MongoClientConfig config) + public MongoSession(TypeManager typeManager, Supplier client, MongoClientConfig config) { this.typeManager = requireNonNull(typeManager, "typeManager is null"); this.client = requireNonNull(client, "client is null"); @@ -200,12 +201,13 @@ public MongoSession(TypeManager typeManager, MongoClient client, MongoClientConf @Override public void close() { - client.close(); + // Close the shared MongoClient instance that was created lazily + getClient().close(); } public List getAddresses() { - return client.getClusterDescription().getServerDescriptions().stream() + return getClient().getClusterDescription().getServerDescriptions().stream() .map(description -> fromParts(description.getAddress().getHost(), description.getAddress().getPort())) .collect(toImmutableList()); } @@ -221,12 +223,12 @@ public List getAllSchemas() public void createSchema(String schemaName) { // Put an empty schema collection because MongoDB doesn't support a database without collections - client.getDatabase(schemaName).createCollection(schemaCollection); + getClient().getDatabase(schemaName).createCollection(schemaCollection); } public void dropSchema(String schemaName, boolean cascade) { - MongoDatabase database = client.getDatabase(toRemoteSchemaName(schemaName)); + MongoDatabase database = getClient().getDatabase(toRemoteSchemaName(schemaName)); if (!cascade) { try (MongoCursor collections = database.listCollectionNames().cursor()) { while (collections.hasNext()) { @@ -281,7 +283,7 @@ public void createTable(RemoteTableName name, List columns, O throw new SchemaNotFoundException(name.databaseName()); } createTableMetadata(name, columns, comment); - client.getDatabase(name.databaseName()).createCollection(name.collectionName()); + getClient().getDatabase(name.databaseName()).createCollection(name.collectionName()); } public void dropTable(RemoteTableName remoteTableName) @@ -300,7 +302,7 @@ public void setTableComment(MongoTableHandle table, Optional comment) Document metadata = getTableMetadata(remoteSchemaName, remoteTableName); metadata.append(COMMENT_KEY, comment.orElse(null)); - client.getDatabase(remoteSchemaName).getCollection(schemaCollection) + getClient().getDatabase(remoteSchemaName).getCollection(schemaCollection) .findOneAndReplace(new Document(TABLE_NAME_KEY, remoteTableName), metadata); tableCache.invalidate(table.schemaTableName()); @@ -323,7 +325,7 @@ public void setColumnComment(MongoTableHandle table, String columnName, Optional metadata.append(FIELDS_KEY, columns.build()); - client.getDatabase(remoteSchemaName).getCollection(schemaCollection) + getClient().getDatabase(remoteSchemaName).getCollection(schemaCollection) .findOneAndReplace(new Document(TABLE_NAME_KEY, remoteTableName), metadata); tableCache.invalidate(table.schemaTableName()); @@ -336,16 +338,16 @@ public void renameTable(MongoTableHandle table, SchemaTableName newName) String newSchemaName = toRemoteSchemaName(newName.getSchemaName()); // Schema collection should always have the source table definition - MongoCollection oldSchema = client.getDatabase(oldSchemaName).getCollection(schemaCollection); + MongoCollection oldSchema = getClient().getDatabase(oldSchemaName).getCollection(schemaCollection); Document tableDefinition = oldSchema.findOneAndDelete(new Document(TABLE_NAME_KEY, oldTableName)); requireNonNull(tableDefinition, "Table definition not found in schema collection: " + oldTableName); - MongoCollection newSchema = client.getDatabase(newSchemaName).getCollection(schemaCollection); + MongoCollection newSchema = getClient().getDatabase(newSchemaName).getCollection(schemaCollection); tableDefinition.append(TABLE_NAME_KEY, newName.getTableName()); newSchema.insertOne(tableDefinition); // Need to check explicitly because the old collection may not exist when it doesn't have any data - if (collectionExists(client.getDatabase(oldSchemaName), oldTableName)) { + if (collectionExists(getClient().getDatabase(oldSchemaName), oldTableName)) { getCollection(table.remoteTableName()).renameCollection(new MongoNamespace(newSchemaName, newName.getTableName())); } @@ -370,7 +372,7 @@ public void addColumn(MongoTableHandle table, ColumnMetadata columnMetadata) metadata.append(FIELDS_KEY, columns); - MongoDatabase db = client.getDatabase(remoteSchemaName); + MongoDatabase db = getClient().getDatabase(remoteSchemaName); MongoCollection schema = db.getCollection(schemaCollection); schema.findOneAndReplace(new Document(TABLE_NAME_KEY, remoteTableName), metadata); @@ -395,7 +397,7 @@ public void renameColumn(MongoTableHandle table, String source, String target) metadata.append(FIELDS_KEY, columns); - MongoDatabase database = client.getDatabase(remoteSchemaName); + MongoDatabase database = getClient().getDatabase(remoteSchemaName); MongoCollection schema = database.getCollection(schemaCollection); schema.findOneAndReplace(new Document(TABLE_NAME_KEY, remoteTableName), metadata); @@ -418,7 +420,7 @@ public void dropColumn(MongoTableHandle table, String columnName) metadata.append(FIELDS_KEY, columns); - MongoDatabase database = client.getDatabase(remoteSchemaName); + MongoDatabase database = getClient().getDatabase(remoteSchemaName); MongoCollection schema = database.getCollection(schemaCollection); schema.findOneAndReplace(new Document(TABLE_NAME_KEY, remoteTableName), metadata); @@ -447,7 +449,7 @@ public void setColumnType(MongoTableHandle table, String columnName, Type type) metadata.replace(FIELDS_KEY, columns); - client.getDatabase(remoteSchemaName).getCollection(schemaCollection) + getClient().getDatabase(remoteSchemaName).getCollection(schemaCollection) .findOneAndReplace(new Document(TABLE_NAME_KEY, remoteTableName), metadata); tableCache.invalidate(table.schemaTableName()); @@ -501,7 +503,7 @@ private static Optional getComment(Document doc) public MongoCollection getCollection(RemoteTableName remoteTableName) { - return client.getDatabase(remoteTableName.databaseName()).getCollection(remoteTableName.collectionName()); + return getClient().getDatabase(remoteTableName.databaseName()).getCollection(remoteTableName.collectionName()); } public List getIndexes(String schemaName, String tableName) @@ -509,7 +511,7 @@ public List getIndexes(String schemaName, String tableName) if (isView(schemaName, tableName)) { return ImmutableList.of(); } - MongoCollection collection = client.getDatabase(schemaName).getCollection(tableName); + MongoCollection collection = getClient().getDatabase(schemaName).getCollection(tableName); return MongoIndex.parse(collection.listIndexes()); } @@ -801,7 +803,7 @@ private static Document isNotNullPredicate() private Document getTableMetadata(String schemaName, String tableName) throws TableNotFoundException { - MongoDatabase db = client.getDatabase(schemaName); + MongoDatabase db = getClient().getDatabase(schemaName); MongoCollection schema = db.getCollection(schemaCollection); Document doc = schema @@ -843,7 +845,7 @@ private boolean indexExists(MongoCollection schemaCollection) private Set getTableMetadataNames(String schemaName) { - try (MongoCursor cursor = client.getDatabase(schemaName).getCollection(schemaCollection) + try (MongoCursor cursor = getClient().getDatabase(schemaName).getCollection(schemaCollection) .find().projection(new Document(TABLE_NAME_KEY, true)).iterator()) { return Streams.stream(cursor) .map(document -> document.getString(TABLE_NAME_KEY)) @@ -856,7 +858,7 @@ private void createTableMetadata(RemoteTableName remoteSchemaTableName, List fields = new ArrayList<>(); @@ -881,7 +883,7 @@ private void createTableMetadata(RemoteTableName remoteSchemaTableName, List guessTableFields(String schemaName, String tableName) { - MongoDatabase db = client.getDatabase(schemaName); + MongoDatabase db = getClient().getDatabase(schemaName); Document doc = db.getCollection(tableName).find().first(); if (doc == null || doc.isEmpty()) { // no records at the collection @@ -1044,7 +1046,7 @@ private String toRemoteSchemaName(String schemaName) private MongoIterable listDatabaseNames() { - return client.listDatabases() + return getClient().listDatabases() .nameOnly(true) .authorizedDatabasesOnly(true) .map(result -> result.getString("name")); @@ -1066,7 +1068,7 @@ private String toRemoteTableName(String schemaName, String tableName) private List listCollectionNames(String databaseName) { - MongoDatabase database = client.getDatabase(databaseName); + MongoDatabase database = getClient().getDatabase(databaseName); Document cursor = database.runCommand(new Document(AUTHORIZED_LIST_COLLECTIONS_COMMAND)).get("cursor", Document.class); List firstBatch = cursor.get("firstBatch", List.class); @@ -1083,7 +1085,7 @@ private boolean isView(String schemaName, String tableName) .put("nameOnly", true) .put("authorizedCollections", true) .buildOrThrow()); - Document cursor = client.getDatabase(schemaName).runCommand(listCollectionsCommand).get("cursor", Document.class); + Document cursor = getClient().getDatabase(schemaName).runCommand(listCollectionsCommand).get("cursor", Document.class); List firstBatch = cursor.get("firstBatch", List.class); if (firstBatch.isEmpty()) { return false; @@ -1091,4 +1093,9 @@ private boolean isView(String schemaName, String tableName) String type = firstBatch.get(0).getString("type"); return "view".equals(type); } + + private MongoClient getClient() + { + return client.get(); + } } diff --git a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/MongoServer.java b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/MongoServer.java index febb162ce07a..023f1191e98d 100644 --- a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/MongoServer.java +++ b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/MongoServer.java @@ -14,7 +14,7 @@ package io.trino.plugin.mongodb; import com.mongodb.ConnectionString; -import org.testcontainers.containers.MongoDBContainer; +import org.testcontainers.mongodb.MongoDBContainer; import java.io.Closeable; diff --git a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java index 6d7147ac8287..f97bf8691710 100644 --- a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java +++ b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoConnectorTest.java @@ -102,6 +102,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) return switch (connectorBehavior) { case SUPPORTS_ADD_COLUMN_WITH_POSITION, SUPPORTS_ADD_FIELD, + SUPPORTS_AGGREGATION_PUSHDOWN, SUPPORTS_CREATE_MATERIALIZED_VIEW, SUPPORTS_CREATE_VIEW, SUPPORTS_DEFAULT_COLUMN_VALUE, @@ -111,6 +112,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_RENAME_FIELD, SUPPORTS_RENAME_SCHEMA, SUPPORTS_SET_FIELD_TYPE, + SUPPORTS_TOPN_PUSHDOWN, SUPPORTS_TRUNCATE, SUPPORTS_UPDATE -> false; default -> super.hasBehavior(connectorBehavior); @@ -980,10 +982,8 @@ public void testNullPredicates() } @Test - public void testLimitPushdown() + void testLimitWithLowerAndUpperBound() { - assertThat(query("SELECT name FROM nation LIMIT 30")).isFullyPushedDown(); // Use high limit for result determinism - // Make sure LIMIT 0 returns empty result because cursor.limit(0) means no limit in MongoDB assertThat(query("SELECT name FROM nation LIMIT 0")).returnsEmptyResult(); diff --git a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoPlugin.java b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoPlugin.java index f71fae5fcc34..bac1f9f589a9 100644 --- a/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoPlugin.java +++ b/plugin/trino-mongodb/src/test/java/io/trino/plugin/mongodb/TestMongoPlugin.java @@ -44,4 +44,17 @@ public void testCreateConnector() connector.shutdown(); } + + @Test + public void testUnreachableMongoDbSrv() + { + ConnectorFactory factory = getOnlyElement(new MongoPlugin().getConnectorFactories()); + factory.create( + "test", + ImmutableMap.of( + "mongodb.connection-url", "mongodb+srv://localhost", + "bootstrap.quiet", "true"), + new TestingConnectorContext()) + .shutdown(); + } } diff --git a/plugin/trino-mysql-event-listener/pom.xml b/plugin/trino-mysql-event-listener/pom.xml index 4e906c308fac..9c0f36fb7fe4 100644 --- a/plugin/trino-mysql-event-listener/pom.xml +++ b/plugin/trino-mysql-event-listener/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -155,7 +155,7 @@ org.testcontainers - mysql + testcontainers-mysql test diff --git a/plugin/trino-mysql-event-listener/src/main/java/io/trino/plugin/eventlistener/mysql/MysqlEventListenerFactory.java b/plugin/trino-mysql-event-listener/src/main/java/io/trino/plugin/eventlistener/mysql/MysqlEventListenerFactory.java index 000e16df9dc8..d64e21ac74d6 100644 --- a/plugin/trino-mysql-event-listener/src/main/java/io/trino/plugin/eventlistener/mysql/MysqlEventListenerFactory.java +++ b/plugin/trino-mysql-event-listener/src/main/java/io/trino/plugin/eventlistener/mysql/MysqlEventListenerFactory.java @@ -72,6 +72,7 @@ public EventListener create(Map config, EventListenerContext con Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-mysql-event-listener/src/test/java/io/trino/plugin/eventlistener/mysql/TestMysqlEventListener.java b/plugin/trino-mysql-event-listener/src/test/java/io/trino/plugin/eventlistener/mysql/TestMysqlEventListener.java index c52d42071c6b..5529017456db 100644 --- a/plugin/trino-mysql-event-listener/src/test/java/io/trino/plugin/eventlistener/mysql/TestMysqlEventListener.java +++ b/plugin/trino-mysql-event-listener/src/test/java/io/trino/plugin/eventlistener/mysql/TestMysqlEventListener.java @@ -40,7 +40,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.parallel.Execution; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import java.net.URI; import java.sql.Connection; @@ -106,6 +106,7 @@ final class TestMysqlEventListener Optional.of(ofMillis(113)), Optional.of(ofMillis(114)), Optional.of(ofMillis(115)), + Optional.of(ofMillis(116)), 115L, 116L, 117L, @@ -141,6 +142,8 @@ final class TestMysqlEventListener // not stored Collections.emptyList(), // not stored + ImmutableMap.of(), + // not stored Optional.empty()); private static final QueryContext FULL_QUERY_CONTEXT = new QueryContext( @@ -272,6 +275,7 @@ final class TestMysqlEventListener Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), 115L, 116L, 117L, @@ -304,6 +308,7 @@ final class TestMysqlEventListener Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + ImmutableMap.of(), // not stored Optional.empty()); @@ -348,7 +353,7 @@ final class TestMysqlEventListener Instant.now(), Instant.now()); - private MySQLContainer mysqlContainer; + private MySQLContainer mysqlContainer; private String mysqlContainerUrl; private EventListener eventListener; private JsonCodecFactory jsonCodecFactory; @@ -356,7 +361,7 @@ final class TestMysqlEventListener @BeforeAll void setup() { - mysqlContainer = new MySQLContainer<>("mysql:8.0.36"); + mysqlContainer = new MySQLContainer("mysql:8.0.36"); mysqlContainer.start(); mysqlContainerUrl = getJdbcUrl(mysqlContainer); eventListener = new MysqlEventListenerFactory() @@ -376,7 +381,7 @@ void teardown() jsonCodecFactory = null; } - private static String getJdbcUrl(MySQLContainer container) + private static String getJdbcUrl(MySQLContainer container) { return format("%s?user=%s&password=%s&useSSL=false&allowPublicKeyRetrieval=true", container.getJdbcUrl(), diff --git a/plugin/trino-mysql/pom.xml b/plugin/trino-mysql/pom.xml index aa209522f7cf..fc39d17f40b9 100644 --- a/plugin/trino-mysql/pom.xml +++ b/plugin/trino-mysql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -239,13 +239,13 @@ org.testcontainers - mysql + testcontainers test org.testcontainers - testcontainers + testcontainers-mysql test diff --git a/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/TestingMySqlServer.java b/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/TestingMySqlServer.java index c83555f5b127..72421e590819 100644 --- a/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/TestingMySqlServer.java +++ b/plugin/trino-mysql/src/test/java/io/trino/plugin/mysql/TestingMySqlServer.java @@ -13,7 +13,7 @@ */ package io.trino.plugin.mysql; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import java.io.Closeable; import java.io.IOException; @@ -27,7 +27,7 @@ import static io.trino.testing.containers.TestContainers.startOrReuse; import static java.lang.String.format; import static java.time.ZoneOffset.UTC; -import static org.testcontainers.containers.MySQLContainer.MYSQL_PORT; +import static org.testcontainers.mysql.MySQLContainer.MYSQL_PORT; public class TestingMySqlServer implements AutoCloseable @@ -36,7 +36,7 @@ public class TestingMySqlServer public static final String DEFAULT_IMAGE = DEFAULT_IMAGE_8; public static final String LEGACY_IMAGE = "mysql:5.7.44"; // oldest available on RDS - private final MySQLContainer container; + private final MySQLContainer container; private final Closeable cleanup; public TestingMySqlServer() @@ -61,7 +61,7 @@ public TestingMySqlServer(String dockerImageName, boolean globalTransactionEnabl public TestingMySqlServer(String dockerImageName, boolean globalTransactionEnable, ZoneId zoneId) { - MySQLContainer container = new MySQLContainer<>(dockerImageName); + MySQLContainer container = new MySQLContainer(dockerImageName); container = container.withDatabaseName("tpch"); container.addEnv("TZ", zoneId.getId()); if (globalTransactionEnable) { @@ -80,7 +80,7 @@ public TestingMySqlServer(String dockerImageName, boolean globalTransactionEnabl } } - private void configureContainer(MySQLContainer container) + private void configureContainer(MySQLContainer container) { // MySQL configuration provided by default by testcontainers causes MySQL to produce poor estimates in CARDINALITY column of INFORMATION_SCHEMA.STATISTICS table. container.addParameter("TC_MY_CNF", null); diff --git a/plugin/trino-opa/pom.xml b/plugin/trino-opa/pom.xml index aeb203c95afd..c6857db38a5e 100644 --- a/plugin/trino-opa/pom.xml +++ b/plugin/trino-opa/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -150,6 +150,7 @@ trino-testing test + org.assertj assertj-core @@ -188,15 +189,14 @@ org.testcontainers - junit-jupiter + testcontainers test + org.testcontainers - testcontainers + testcontainers-junit-jupiter test - - diff --git a/plugin/trino-opa/src/main/java/io/trino/plugin/opa/OpaAccessControl.java b/plugin/trino-opa/src/main/java/io/trino/plugin/opa/OpaAccessControl.java index a0b92afee640..046a812c2c8c 100644 --- a/plugin/trino-opa/src/main/java/io/trino/plugin/opa/OpaAccessControl.java +++ b/plugin/trino-opa/src/main/java/io/trino/plugin/opa/OpaAccessControl.java @@ -802,11 +802,11 @@ private static Map> convertProperties(Map config, Optional Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); return injector.getInstance(SystemAccessControl.class); diff --git a/plugin/trino-opa/src/main/java/io/trino/plugin/opa/schema/OpaQueryContext.java b/plugin/trino-opa/src/main/java/io/trino/plugin/opa/schema/OpaQueryContext.java index 22c981135126..66fa7304af4a 100644 --- a/plugin/trino-opa/src/main/java/io/trino/plugin/opa/schema/OpaQueryContext.java +++ b/plugin/trino-opa/src/main/java/io/trino/plugin/opa/schema/OpaQueryContext.java @@ -13,13 +13,18 @@ */ package io.trino.plugin.opa.schema; +import io.trino.spi.QueryId; + +import java.util.Optional; + import static java.util.Objects.requireNonNull; -public record OpaQueryContext(TrinoIdentity identity, OpaPluginContext softwareStack) +public record OpaQueryContext(TrinoIdentity identity, OpaPluginContext softwareStack, Optional queryId) { public OpaQueryContext { requireNonNull(identity, "identity is null"); requireNonNull(softwareStack, "softwareStack is null"); + requireNonNull(queryId, "queryId is null"); } } diff --git a/plugin/trino-opa/src/test/java/io/trino/plugin/opa/TestOpaAccessControl.java b/plugin/trino-opa/src/test/java/io/trino/plugin/opa/TestOpaAccessControl.java index 4740ebdf94e2..e2c795b4a92d 100644 --- a/plugin/trino-opa/src/test/java/io/trino/plugin/opa/TestOpaAccessControl.java +++ b/plugin/trino-opa/src/test/java/io/trino/plugin/opa/TestOpaAccessControl.java @@ -24,6 +24,7 @@ import io.trino.plugin.opa.HttpClientUtils.InstrumentedHttpClient; import io.trino.plugin.opa.HttpClientUtils.MockResponse; import io.trino.plugin.opa.schema.OpaViewExpression; +import io.trino.spi.QueryId; import io.trino.spi.connector.CatalogSchemaName; import io.trino.spi.connector.CatalogSchemaRoutineName; import io.trino.spi.connector.CatalogSchemaTableName; @@ -37,6 +38,7 @@ import io.trino.spi.type.VarcharType; import org.junit.jupiter.api.Test; +import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Optional; @@ -960,6 +962,51 @@ void testGetColumnMasksThrowsForIllegalResponse() "Failed to deserialize"); } + + @Test + public void testQueryIdPropagation() + { + QueryId queryId = new QueryId("20250718_081710_03427_trino"); + + SystemSecurityContext customSecurityContext = new SystemSecurityContext(TEST_IDENTITY, queryId, Instant.now()); + CatalogSchemaTableName tableName = new CatalogSchemaTableName("my_catalog", "my_schema", "my_table"); + + ThrowingMethodWrapper wrappedMethod = new ThrowingMethodWrapper(accessControl -> + accessControl.checkCanShowCreateTable(customSecurityContext, tableName)); + + String expectedActionRequest = + """ + { + "operation": "ShowCreateTable", + "resource": { + "table": { + "catalogName": "%s", + "schemaName": "%s", + "tableName": "%s" + } + } + } + """.formatted( + tableName.getCatalogName(), + tableName.getSchemaTableName().getSchemaName(), + tableName.getSchemaTableName().getTableName()); + + InstrumentedHttpClient mockClient = createMockHttpClient(OPA_SERVER_URI, request -> { + JsonNode contextNode = request.path("input").path("context"); + + assertThat(contextNode.path("queryId").asText()).isEqualTo(queryId.id()); + assertThat(contextNode.path("identity").path("user").asText()).isEqualTo(TEST_IDENTITY.getUser()); + assertThat(contextNode.path("softwareStack").path("trinoVersion").asText()).isEqualTo("trino-version"); + + return OK_RESPONSE; + }); + + OpaAccessControl authorizer = createOpaAuthorizer(simpleOpaConfig(), mockClient); + + assertThat(wrappedMethod.isAccessAllowed(authorizer)).isTrue(); + assertStringRequestsEqual(ImmutableSet.of(expectedActionRequest), mockClient.getRequests(), "/input/action"); + } + private void testGetColumnMasks(Map columnResponseContent, Map expectedResult) { InstrumentedHttpClient httpClient = createMockHttpClient( diff --git a/plugin/trino-openlineage/pom.xml b/plugin/trino-openlineage/pom.xml index fb0eaa892658..ade4f992b089 100644 --- a/plugin/trino-openlineage/pom.xml +++ b/plugin/trino-openlineage/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -63,7 +63,7 @@ io.openlineage openlineage-java - 1.37.0 + 1.39.0 @@ -234,13 +234,13 @@ org.testcontainers - postgresql + testcontainers test org.testcontainers - testcontainers + testcontainers-postgresql test diff --git a/plugin/trino-openlineage/src/main/java/io/trino/plugin/openlineage/OpenLineageListenerFactory.java b/plugin/trino-openlineage/src/main/java/io/trino/plugin/openlineage/OpenLineageListenerFactory.java index 4b9a6d5dd7f4..1b493bda8ca1 100644 --- a/plugin/trino-openlineage/src/main/java/io/trino/plugin/openlineage/OpenLineageListenerFactory.java +++ b/plugin/trino-openlineage/src/main/java/io/trino/plugin/openlineage/OpenLineageListenerFactory.java @@ -35,6 +35,7 @@ public EventListener create(Map config, EventListenerContext con Bootstrap app = new Bootstrap("io.trino.bootstrap.listener." + getName(), new OpenLineageListenerModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/BaseTestOpenLineageQueries.java b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/BaseTestOpenLineageQueries.java index 3a2aabc50aa7..5f98b0b5844a 100644 --- a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/BaseTestOpenLineageQueries.java +++ b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/BaseTestOpenLineageQueries.java @@ -504,19 +504,19 @@ public enum LineageTestTableType "TABLE", QueryType.INSERT, true, - 44, + 46, true), VIEW( "VIEW", QueryType.DATA_DEFINITION, false, - 43, + 45, false), MATERIALIZED_VIEW( "MATERIALIZED VIEW", QueryType.DATA_DEFINITION, false, - 43, + 45, false); private final String queryReplacement; diff --git a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/MarquezServer.java b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/MarquezServer.java index c1e707c0e0e3..48a66954b7df 100644 --- a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/MarquezServer.java +++ b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/MarquezServer.java @@ -16,8 +16,8 @@ import com.google.common.io.Closer; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testcontainers.utility.MountableFile; import java.io.Closeable; @@ -62,7 +62,7 @@ public class MarquezServer } private final GenericContainer dockerContainerAPI; - private final PostgreSQLContainer dockerContainerPostgres; + private final PostgreSQLContainer dockerContainerPostgres; private final GenericContainer dockerWebUIContainerAPI; public MarquezServer() @@ -75,7 +75,7 @@ public MarquezServer(String version) network = Network.newNetwork(); closer.register(this.network::close); - this.dockerContainerPostgres = new PostgreSQLContainer<>("postgres:14") + this.dockerContainerPostgres = new PostgreSQLContainer("postgres:14") .withNetwork(network) .withNetworkAliases(POSTGRES_HOST) .withDatabaseName(POSTGRES_DB) diff --git a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TrinoEventData.java b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TrinoEventData.java index e887e2732e53..9a9948e75dd7 100644 --- a/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TrinoEventData.java +++ b/plugin/trino-openlineage/src/test/java/io/trino/plugin/openlineage/TrinoEventData.java @@ -13,6 +13,7 @@ */ package io.trino.plugin.openlineage; +import com.google.common.collect.ImmutableMap; import io.trino.operator.RetryPolicy; import io.trino.spi.eventlistener.QueryCompletedEvent; import io.trino.spi.eventlistener.QueryContext; @@ -111,6 +112,7 @@ private TrinoEventData() Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), 0L, 0L, 0L, @@ -137,6 +139,7 @@ private TrinoEventData() Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), + ImmutableMap.of(), Optional.empty()); queryCompleteEvent = new QueryCompletedEvent( diff --git a/plugin/trino-opensearch/pom.xml b/plugin/trino-opensearch/pom.xml index a85e6a0e1462..3c3123a3e203 100644 --- a/plugin/trino-opensearch/pom.xml +++ b/plugin/trino-opensearch/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -180,6 +180,11 @@ aws-core + + software.amazon.awssdk + http-auth-aws + + software.amazon.awssdk http-client-spi @@ -386,7 +391,7 @@ org.opensearch opensearch-testcontainers - 3.0.2 + 4.0.0 test diff --git a/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/OpenSearchConnectorFactory.java b/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/OpenSearchConnectorFactory.java index 5732a8eea9cf..c0d2b33b35df 100644 --- a/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/OpenSearchConnectorFactory.java +++ b/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/OpenSearchConnectorFactory.java @@ -16,11 +16,10 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.spi.Node; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; @@ -55,15 +54,13 @@ public Connector create(String catalogName, Map config, Connecto new MBeanServerModule(), new ConnectorObjectNameGeneratorModule("io.trino.plugin.opensearch", "trino.plugin.opensearch"), new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), new OpenSearchConnectorModule(), - binder -> { - binder.bind(Node.class).toInstance(context.getCurrentNode()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }); + new ConnectorContextModule(catalogName, context)); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/client/AwsRequestSigner.java b/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/client/AwsRequestSigner.java index ed561d4f8b21..a3af19d9b7ee 100644 --- a/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/client/AwsRequestSigner.java +++ b/plugin/trino-opensearch/src/main/java/io/trino/plugin/opensearch/client/AwsRequestSigner.java @@ -26,13 +26,13 @@ import org.apache.http.protocol.HttpContext; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.signer.Aws4Signer; -import software.amazon.awssdk.auth.signer.internal.SignerConstant; import software.amazon.awssdk.auth.signer.params.Aws4SignerParams; import software.amazon.awssdk.auth.signer.params.SignerChecksumParams; import software.amazon.awssdk.core.checksums.Algorithm; import software.amazon.awssdk.http.ContentStreamProvider; import software.amazon.awssdk.http.SdkHttpFullRequest; import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.auth.aws.signer.SignerConstant; import software.amazon.awssdk.regions.Region; import java.io.IOException; diff --git a/plugin/trino-opensearch/src/test/java/io/trino/plugin/opensearch/BaseOpenSearchConnectorTest.java b/plugin/trino-opensearch/src/test/java/io/trino/plugin/opensearch/BaseOpenSearchConnectorTest.java index 13ad0104ecd9..eaa2aeb17118 100644 --- a/plugin/trino-opensearch/src/test/java/io/trino/plugin/opensearch/BaseOpenSearchConnectorTest.java +++ b/plugin/trino-opensearch/src/test/java/io/trino/plugin/opensearch/BaseOpenSearchConnectorTest.java @@ -19,7 +19,6 @@ import com.google.common.net.HostAndPort; import io.trino.Session; import io.trino.spi.type.VarcharType; -import io.trino.sql.planner.plan.LimitNode; import io.trino.sql.planner.plan.ProjectNode; import io.trino.testing.AbstractTestQueries; import io.trino.testing.BaseConnectorTest; @@ -1679,13 +1678,6 @@ public void testFiltersCharset() .matches("VALUES (VARCHAR 'Türkiye')"); } - @Test - public void testLimitPushdown() - throws IOException - { - assertThat(query("SELECT name FROM nation LIMIT 30")).isNotFullyPushedDown(LimitNode.class); // Use high limit for result determinism - } - @Test public void testDataTypesNested() throws IOException diff --git a/plugin/trino-oracle/pom.xml b/plugin/trino-oracle/pom.xml index d186099a3d31..88ac21fd339d 100644 --- a/plugin/trino-oracle/pom.xml +++ b/plugin/trino-oracle/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -251,13 +251,19 @@ org.testcontainers - oracle-xe + testcontainers test org.testcontainers - testcontainers + testcontainers-jdbc + test + + + + org.testcontainers + testcontainers-oracle-free test diff --git a/plugin/trino-oracle/src/main/java/io/trino/plugin/oracle/OracleClient.java b/plugin/trino-oracle/src/main/java/io/trino/plugin/oracle/OracleClient.java index 0432c37f6494..136121d43971 100644 --- a/plugin/trino-oracle/src/main/java/io/trino/plugin/oracle/OracleClient.java +++ b/plugin/trino-oracle/src/main/java/io/trino/plugin/oracle/OracleClient.java @@ -620,7 +620,7 @@ private static Optional toTypeHandle(DecimalType decimalType) @Override protected Optional> limitFunction() { - return Optional.of((sql, limit) -> format("SELECT * FROM (%s) WHERE ROWNUM <= %s", sql, limit)); + return Optional.of((sql, limit) -> format("%s FETCH FIRST %s ROWS ONLY", sql, limit)); } @Override diff --git a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java index 435a0fc2f07f..7a8f7d595466 100644 --- a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java +++ b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/BaseOracleConnectorTest.java @@ -137,7 +137,7 @@ public void testInformationSchemaFiltering() @Override protected boolean isColumnNameRejected(Exception exception, String columnName, boolean delimited) { - if (columnName.equals("a\"quote") && exception.getMessage().contains("ORA-03001: unimplemented feature")) { + if (columnName.equals("a\"quote") && exception.getMessage().contains("ORA-25716: The identifier contains a double quotation mark (\") character")) { return true; } @@ -375,6 +375,14 @@ public void testTooLargeDomainCompactionThreshold() "SELECT * from nation", "Domain compaction threshold \\(10000\\) cannot exceed 1000"); } + @Test + @Override // Override because Oracle allows SELECT query in execute procedure + public void testExecuteProcedureWithInvalidQuery() + { + assertUpdate("CALL system.execute('SELECT 1')"); + assertQueryFails("CALL system.execute('invalid')", "(?s)Failed to execute query.*"); + } + @Test @Override public void testNativeQuerySimple() @@ -444,37 +452,37 @@ protected void verifyConcurrentAddColumnFailurePermissible(Exception e) @Override protected OptionalInt maxSchemaNameLength() { - return OptionalInt.of(30); + return OptionalInt.of(128); } @Override protected void verifySchemaNameLengthFailurePermissible(Throwable e) { - assertThat(e).hasMessageContaining("ORA-00972: identifier is too long"); + assertThat(e).hasMessageContaining("ORA-00972"); } @Override protected OptionalInt maxTableNameLength() { - return OptionalInt.of(30); + return OptionalInt.of(128); } @Override protected void verifyTableNameLengthFailurePermissible(Throwable e) { - assertThat(e).hasMessageContaining("ORA-00972: identifier is too long"); + assertThat(e).hasMessageContaining("ORA-00972"); } @Override protected OptionalInt maxColumnNameLength() { - return OptionalInt.of(30); + return OptionalInt.of(128); } @Override protected void verifyColumnNameLengthFailurePermissible(Throwable e) { - assertThat(e).hasMessageContaining("ORA-00972: identifier is too long"); + assertThat(e).hasMessageContaining("ORA-00972"); } @Override diff --git a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/TestingOracleServer.java b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/TestingOracleServer.java index 8b089c5ce314..2f0f8d762d5d 100644 --- a/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/TestingOracleServer.java +++ b/plugin/trino-oracle/src/test/java/io/trino/plugin/oracle/TestingOracleServer.java @@ -24,7 +24,8 @@ import io.trino.plugin.jdbc.credential.StaticCredentialProvider; import io.trino.plugin.jdbc.jmx.StatisticsAwareConnectionFactory; import oracle.jdbc.OracleDriver; -import org.testcontainers.containers.OracleContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.oracle.OracleContainer; import org.testcontainers.utility.MountableFile; import java.io.Closeable; @@ -36,6 +37,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; +import java.time.Duration; import java.time.temporal.ChronoUnit; import static io.trino.testing.TestingConnectorSession.SESSION; @@ -73,11 +75,12 @@ public TestingOracleServer() private void createContainer() { - OracleContainer container = new OracleContainer("gvenzl/oracle-xe:11.2.0.2-full") + OracleContainer container = new OracleContainer("gvenzl/oracle-free:23.9-slim") .withCopyFileToContainer(MountableFile.forClasspathResource("init.sql"), "/container-entrypoint-initdb.d/01-init.sql") .withCopyFileToContainer(MountableFile.forClasspathResource("restart.sh"), "/container-entrypoint-initdb.d/02-restart.sh") .withCopyFileToContainer(MountableFile.forHostPath(createConfigureScript()), "/container-entrypoint-initdb.d/03-create-users.sql") - .usingSid(); + .waitingFor(Wait.forLogMessage(".*DATABASE IS READY TO USE!.*\\s", 1).withStartupTimeout(Duration.ofMinutes(2))) + .withStartupTimeoutSeconds(180); try { this.cleanup = startOrReuse(container); this.container = container; @@ -95,6 +98,7 @@ private Path createConfigureScript() File tempFile = File.createTempFile("init-", ".sql"); Files.write(Joiner.on("\n").join( + format("ALTER SESSION SET CONTAINER=FREEPDB1;"), format("CREATE TABLESPACE %s DATAFILE 'test_db.dat' SIZE 100M ONLINE;", TEST_TABLESPACE), format("CREATE USER %s IDENTIFIED BY %s DEFAULT TABLESPACE %s;", TEST_USER, TEST_PASS, TEST_TABLESPACE), format("GRANT UNLIMITED TABLESPACE TO %s;", TEST_USER), diff --git a/plugin/trino-password-authenticators/pom.xml b/plugin/trino-password-authenticators/pom.xml index 5922c2dbed24..ab07a5432815 100644 --- a/plugin/trino-password-authenticators/pom.xml +++ b/plugin/trino-password-authenticators/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -170,7 +170,7 @@ org.testcontainers - toxiproxy + testcontainers-toxiproxy test diff --git a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileAuthenticatorFactory.java b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileAuthenticatorFactory.java index d0eec948099c..630eb6077514 100644 --- a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileAuthenticatorFactory.java +++ b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileAuthenticatorFactory.java @@ -43,6 +43,7 @@ public PasswordAuthenticator create(Map config) Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileGroupProviderFactory.java b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileGroupProviderFactory.java index e5d6b97f675d..dc8efb2e55f3 100644 --- a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileGroupProviderFactory.java +++ b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/file/FileGroupProviderFactory.java @@ -43,6 +43,7 @@ public GroupProvider create(Map config) Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/ldap/LdapAuthenticatorFactory.java b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/ldap/LdapAuthenticatorFactory.java index 04b080e6ef5a..a298713b1ad5 100644 --- a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/ldap/LdapAuthenticatorFactory.java +++ b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/ldap/LdapAuthenticatorFactory.java @@ -46,6 +46,7 @@ public PasswordAuthenticator create(Map config) Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/salesforce/SalesforceAuthenticatorFactory.java b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/salesforce/SalesforceAuthenticatorFactory.java index 7d27eb1f532e..46460de74f5d 100644 --- a/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/salesforce/SalesforceAuthenticatorFactory.java +++ b/plugin/trino-password-authenticators/src/main/java/io/trino/plugin/password/salesforce/SalesforceAuthenticatorFactory.java @@ -46,6 +46,7 @@ public PasswordAuthenticator create(Map config) Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-pinot/pom.xml b/plugin/trino-pinot/pom.xml old mode 100755 new mode 100644 index 64fd58a53beb..4ecc5c3b2e29 --- a/plugin/trino-pinot/pom.xml +++ b/plugin/trino-pinot/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -26,12 +26,12 @@ net.openhft chronicle-core - 2.27ea1 + 2.27ea8 net.openhft posix - 2.27ea0 + 2.27ea3 diff --git a/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotConnectorFactory.java b/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotConnectorFactory.java index 100154a3101e..ab3b21bbe1a2 100755 --- a/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotConnectorFactory.java +++ b/plugin/trino-pinot/src/main/java/io/trino/plugin/pinot/PinotConnectorFactory.java @@ -18,6 +18,7 @@ import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.plugin.pinot.auth.PinotAuthenticationModule; @@ -59,7 +60,8 @@ public Connector create(String catalogName, Map config, Connecto .add(new JsonModule()) .add(new MBeanModule()) .add(new MBeanServerModule()) - .add(new TypeDeserializerModule(context.getTypeManager())) + .add(new TypeDeserializerModule()) + .add(new ConnectorContextModule(catalogName, context)) .add(new PinotModule(catalogName)) .add(new PinotAuthenticationModule()); @@ -69,6 +71,7 @@ public Connector create(String catalogName, Map config, Connecto Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-pinot/src/test/java/io/trino/plugin/pinot/TestPinotConnectorTest.java b/plugin/trino-pinot/src/test/java/io/trino/plugin/pinot/TestPinotConnectorTest.java index 5a9702417da8..a337bf12c4f4 100644 --- a/plugin/trino-pinot/src/test/java/io/trino/plugin/pinot/TestPinotConnectorTest.java +++ b/plugin/trino-pinot/src/test/java/io/trino/plugin/pinot/TestPinotConnectorTest.java @@ -59,6 +59,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_VIEW, SUPPORTS_DELETE, SUPPORTS_INSERT, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MAP_TYPE, SUPPORTS_MERGE, SUPPORTS_RENAME_COLUMN, diff --git a/plugin/trino-postgresql/pom.xml b/plugin/trino-postgresql/pom.xml index c3c7ea2b60db..d7435cc6e3d0 100644 --- a/plugin/trino-postgresql/pom.xml +++ b/plugin/trino-postgresql/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -280,13 +280,13 @@ org.testcontainers - postgresql + testcontainers test org.testcontainers - testcontainers + testcontainers-postgresql test diff --git a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestingPostgreSqlServer.java b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestingPostgreSqlServer.java index 9660b59cbe23..7a7292e1220e 100644 --- a/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestingPostgreSqlServer.java +++ b/plugin/trino-postgresql/src/test/java/io/trino/plugin/postgresql/TestingPostgreSqlServer.java @@ -18,8 +18,8 @@ import io.trino.plugin.jdbc.RemoteDatabaseEvent; import io.trino.plugin.jdbc.RemoteLogTracingEvent; import org.intellij.lang.annotations.Language; -import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.output.OutputFrame; +import org.testcontainers.postgresql.PostgreSQLContainer; import org.testcontainers.utility.DockerImageName; import java.io.Closeable; @@ -48,7 +48,7 @@ import static java.lang.String.format; import static java.util.Objects.requireNonNull; import static java.util.function.Predicate.not; -import static org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT; +import static org.testcontainers.postgresql.PostgreSQLContainer.POSTGRESQL_PORT; public class TestingPostgreSqlServer implements AutoCloseable @@ -66,7 +66,7 @@ public class TestingPostgreSqlServer private static final Pattern SQL_QUERY_FIND_PATTERN = Pattern.compile("^(: |/C_\\d: )(.*)"); //In PgSQL cursor queries and non-cursor queries are logged differently private static final String LOG_CANCELLED_STATEMENT_PREFIX = "STATEMENT: "; - private final PostgreSQLContainer dockerContainer; + private final PostgreSQLContainer dockerContainer; private final Set tracingEvents = Sets.newConcurrentHashSet(); private final Closeable cleanup; @@ -89,7 +89,7 @@ public TestingPostgreSqlServer(String dockerImageName, boolean shouldExposeFixed public TestingPostgreSqlServer(DockerImageName dockerImageName, boolean shouldExposeFixedPorts) { - dockerContainer = new PostgreSQLContainer<>(dockerImageName) + dockerContainer = new PostgreSQLContainer(dockerImageName) .withStartupAttempts(3) .withDatabaseName(DATABASE) .withUsername(USER) diff --git a/plugin/trino-prometheus/pom.xml b/plugin/trino-prometheus/pom.xml index 1942cf3b84d7..44414c99bda9 100644 --- a/plugin/trino-prometheus/pom.xml +++ b/plugin/trino-prometheus/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusClient.java b/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusClient.java index 569d8769215f..6b62766c83c7 100644 --- a/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusClient.java +++ b/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusClient.java @@ -174,7 +174,7 @@ public ResponseBody fetchUri(URI uri) Request.Builder requestBuilder = new Request.Builder().url(uri.toString()); try { Response response = httpClient.newCall(requestBuilder.build()).execute(); - if (response.isSuccessful() && response.body() != null) { + if (response.isSuccessful()) { return response.body(); } throw new TrinoException(PROMETHEUS_UNKNOWN_ERROR, "Bad response " + response.code() + " " + response.message()); diff --git a/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusConnectorFactory.java b/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusConnectorFactory.java index c451b6284039..1da069b15af1 100644 --- a/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusConnectorFactory.java +++ b/plugin/trino-prometheus/src/main/java/io/trino/plugin/prometheus/PrometheusConnectorFactory.java @@ -16,6 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -47,11 +48,13 @@ public Connector create(String catalogName, Map requiredConfig, Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), + new ConnectorContextModule(catalogName, context), new PrometheusModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(requiredConfig) .initialize(); diff --git a/plugin/trino-ranger/pom.xml b/plugin/trino-ranger/pom.xml index 70103b0bf52a..f938343174e2 100644 --- a/plugin/trino-ranger/pom.xml +++ b/plugin/trino-ranger/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -15,7 +15,7 @@ Trino - Apache Ranger access control - 25.0.0 + 25.0.1 2.7.0 @@ -308,4 +308,22 @@ test + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + + + diff --git a/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControl.java b/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControl.java index 559327e66f3f..8be22e9fa8b4 100644 --- a/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControl.java +++ b/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControl.java @@ -176,7 +176,7 @@ public void checkCanSetUser(Optional principal, String userName) @Override public void checkCanExecuteQuery(Identity identity, QueryId queryId) { - if (!hasPermission(RangerTrinoResource.forQueryId(queryId.getId()), identity, queryId, EXECUTE, "ExecuteQuery")) { + if (!hasPermission(RangerTrinoResource.forQueryId(queryId.id()), identity, queryId, EXECUTE, "ExecuteQuery")) { denyExecuteQuery(); } } @@ -916,22 +916,22 @@ private RangerTrinoAccessRequest createAccessRequest(RangerTrinoResource resourc private Optional getClientAddress(QueryId queryId) { - return queryId != null ? eventListener.getClientAddress(queryId.getId()) : Optional.empty(); + return queryId != null ? eventListener.getClientAddress(queryId.id()) : Optional.empty(); } private Optional getClientType(QueryId queryId) { - return queryId != null ? eventListener.getClientType(queryId.getId()) : Optional.empty(); + return queryId != null ? eventListener.getClientType(queryId.id()) : Optional.empty(); } private Optional getQueryText(QueryId queryId) { - return queryId != null ? eventListener.getQueryText(queryId.getId()) : Optional.empty(); + return queryId != null ? eventListener.getQueryText(queryId.id()) : Optional.empty(); } private Optional getQueryTime(QueryId queryId) { - return queryId != null ? eventListener.getQueryTime(queryId.getId()) : Optional.empty(); + return queryId != null ? eventListener.getQueryTime(queryId.id()) : Optional.empty(); } private Optional getClientAddress(SystemSecurityContext context) diff --git a/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControlFactory.java b/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControlFactory.java index c65b60f18d2b..7b0e8c41c6be 100644 --- a/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControlFactory.java +++ b/plugin/trino-ranger/src/main/java/io/trino/plugin/ranger/RangerSystemAccessControlFactory.java @@ -48,6 +48,7 @@ public SystemAccessControl create(Map config, SystemAccessContro Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-redis/pom.xml b/plugin/trino-redis/pom.xml index 9dcbd81f2f98..09420dfbdf18 100644 --- a/plugin/trino-redis/pom.xml +++ b/plugin/trino-redis/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -89,7 +89,7 @@ redis.clients jedis - 6.2.0 + 7.0.0 diff --git a/plugin/trino-redis/src/main/java/io/trino/plugin/redis/RedisConnectorFactory.java b/plugin/trino-redis/src/main/java/io/trino/plugin/redis/RedisConnectorFactory.java index 4838142ddcbf..e23228763fb5 100644 --- a/plugin/trino-redis/src/main/java/io/trino/plugin/redis/RedisConnectorFactory.java +++ b/plugin/trino-redis/src/main/java/io/trino/plugin/redis/RedisConnectorFactory.java @@ -18,6 +18,7 @@ import com.google.inject.TypeLiteral; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.TypeDeserializerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -57,7 +58,8 @@ public Connector create(String catalogName, Map config, Connecto Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, new JsonModule(), - new TypeDeserializerModule(context.getTypeManager()), + new TypeDeserializerModule(), + new ConnectorContextModule(catalogName, context), new RedisConnectorModule(), binder -> { if (tableDescriptionSupplier.isPresent()) { @@ -72,6 +74,7 @@ public Connector create(String catalogName, Map config, Connecto Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-redis/src/test/java/io/trino/plugin/redis/BaseRedisConnectorTest.java b/plugin/trino-redis/src/test/java/io/trino/plugin/redis/BaseRedisConnectorTest.java index 8188178744a6..9305cfa27266 100644 --- a/plugin/trino-redis/src/test/java/io/trino/plugin/redis/BaseRedisConnectorTest.java +++ b/plugin/trino-redis/src/test/java/io/trino/plugin/redis/BaseRedisConnectorTest.java @@ -37,6 +37,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_VIEW, SUPPORTS_DELETE, SUPPORTS_INSERT, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MAP_TYPE, SUPPORTS_MERGE, SUPPORTS_RENAME_COLUMN, diff --git a/plugin/trino-redshift/pom.xml b/plugin/trino-redshift/pom.xml index 8f5298ea1667..11d4d1d4f86a 100644 --- a/plugin/trino-redshift/pom.xml +++ b/plugin/trino-redshift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -18,7 +18,7 @@ com.amazon.redshift redshift-jdbc42 - 2.1.0.30 + 2.2.0 diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClient.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClient.java index daaf46f78495..a1e0dd0bb802 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClient.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftClient.java @@ -625,6 +625,12 @@ public Optional toColumnMapping(ConnectorSession session, Connect RedshiftClient::readTime, RedshiftClient::writeTime)); } + if ("binary varying".equals(type.jdbcTypeName().orElse(""))) { + return Optional.of(ColumnMapping.sliceMapping( + VARBINARY, + varbinaryReadFunction(), + varbinaryWriteFunction())); + } switch (type.jdbcType()) { case Types.BIT: // Redshift uses this for booleans @@ -678,12 +684,6 @@ public Optional toColumnMapping(ConnectorSession session, Connect true)); } - case Types.LONGVARBINARY: - return Optional.of(ColumnMapping.sliceMapping( - VARBINARY, - varbinaryReadFunction(), - varbinaryWriteFunction())); - case Types.DATE: return Optional.of(ColumnMapping.longMapping( DATE, diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java index 4d4aeba717a2..445abe90164c 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftConnectorFactory.java @@ -15,17 +15,13 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; -import io.opentelemetry.api.OpenTelemetry; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.jdbc.ExtraCredentialsBasedIdentityCacheMappingModule; import io.trino.plugin.jdbc.JdbcModule; import io.trino.plugin.jdbc.credential.CredentialProviderModule; -import io.trino.spi.Node; -import io.trino.spi.VersionEmbedder; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; import java.util.Map; @@ -49,11 +45,7 @@ public Connector create(String catalogName, Map requiredConfig, Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, - binder -> binder.bind(Node.class).toInstance(context.getCurrentNode()), - binder -> binder.bind(TypeManager.class).toInstance(context.getTypeManager()), - binder -> binder.bind(VersionEmbedder.class).toInstance(context.getVersionEmbedder()), - binder -> binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()), - binder -> binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)), + new ConnectorContextModule(catalogName, context), new JdbcModule(), new CredentialProviderModule(), new ExtraCredentialsBasedIdentityCacheMappingModule(), @@ -61,6 +53,7 @@ public Connector create(String catalogName, Map requiredConfig, Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(requiredConfig) .initialize(); diff --git a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java index bab0f514bb4f..510777c3dbda 100644 --- a/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java +++ b/plugin/trino-redshift/src/main/java/io/trino/plugin/redshift/RedshiftPageSourceProvider.java @@ -104,7 +104,7 @@ private ParquetReader parquetReader(TrinoInputFile inputFile, List { ParquetReaderOptions options = ParquetReaderOptions.defaultOptions(); TrinoParquetDataSource dataSource = new TrinoParquetDataSource(inputFile, options, fileFormatDataSourceStats); - ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize()); + ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, options.getMaxFooterReadSize(), Optional.empty()); MessageType fileSchema = parquetMetadata.getFileMetaData().getSchema(); MessageColumnIO messageColumn = getColumnIO(fileSchema, fileSchema); Map, ColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema); @@ -127,6 +127,7 @@ private ParquetReader parquetReader(TrinoInputFile inputFile, List options, RedshiftParquetPageSource::handleException, Optional.empty(), + Optional.empty(), Optional.empty()); } diff --git a/plugin/trino-resource-group-managers/pom.xml b/plugin/trino-resource-group-managers/pom.xml index 6b61905f53de..233897186b79 100644 --- a/plugin/trino-resource-group-managers/pom.xml +++ b/plugin/trino-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -244,31 +244,31 @@ org.testcontainers - jdbc + testcontainers test org.testcontainers - mysql + testcontainers-jdbc test org.testcontainers - oracle-xe + testcontainers-mysql test org.testcontainers - postgresql + testcontainers-oracle-free test org.testcontainers - testcontainers + testcontainers-postgresql test diff --git a/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/FileResourceGroupConfigurationManagerFactory.java b/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/FileResourceGroupConfigurationManagerFactory.java index 1baba9220454..e69574d3fde2 100644 --- a/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/FileResourceGroupConfigurationManagerFactory.java +++ b/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/FileResourceGroupConfigurationManagerFactory.java @@ -42,6 +42,7 @@ public ResourceGroupConfigurationManager create(Map config, R Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/db/DbResourceGroupConfigurationManagerFactory.java b/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/db/DbResourceGroupConfigurationManagerFactory.java index c55579415a33..7d2c0625f017 100644 --- a/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/db/DbResourceGroupConfigurationManagerFactory.java +++ b/plugin/trino-resource-group-managers/src/main/java/io/trino/plugin/resourcegroups/db/DbResourceGroupConfigurationManagerFactory.java @@ -53,6 +53,7 @@ public ResourceGroupConfigurationManager create(Map config, R Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsMysqlFlywayMigration.java b/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsMysqlFlywayMigration.java index 60d84beaccda..32d1fbd0cc34 100644 --- a/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsMysqlFlywayMigration.java +++ b/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsMysqlFlywayMigration.java @@ -14,7 +14,7 @@ package io.trino.plugin.resourcegroups.db; import org.testcontainers.containers.JdbcDatabaseContainer; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; public class TestDbResourceGroupsMysqlFlywayMigration extends BaseTestDbResourceGroupsFlywayMigration @@ -22,7 +22,7 @@ public class TestDbResourceGroupsMysqlFlywayMigration @Override protected final JdbcDatabaseContainer startContainer() { - JdbcDatabaseContainer container = new MySQLContainer<>("mysql:8.0.36"); + JdbcDatabaseContainer container = new MySQLContainer("mysql:8.0.36"); container.start(); return container; } diff --git a/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsOracleFlywayMigration.java b/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsOracleFlywayMigration.java index de706e7cde9d..e0ae3dd874f3 100644 --- a/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsOracleFlywayMigration.java +++ b/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsOracleFlywayMigration.java @@ -15,7 +15,7 @@ import org.jdbi.v3.core.Handle; import org.testcontainers.containers.JdbcDatabaseContainer; -import org.testcontainers.containers.OracleContainer; +import org.testcontainers.oracle.OracleContainer; import java.sql.DatabaseMetaData; import java.sql.ResultSet; @@ -28,7 +28,7 @@ public class TestDbResourceGroupsOracleFlywayMigration @Override protected final JdbcDatabaseContainer startContainer() { - JdbcDatabaseContainer container = new OracleContainer("gvenzl/oracle-xe:18.4.0-slim") + JdbcDatabaseContainer container = new OracleContainer("gvenzl/oracle-free:23.9-slim") .withPassword("trino") .withEnv("ORACLE_PASSWORD", "trino"); container.start(); diff --git a/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsPostgresqlFlywayMigration.java b/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsPostgresqlFlywayMigration.java index 4de957dc01ee..d424c7421c32 100644 --- a/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsPostgresqlFlywayMigration.java +++ b/plugin/trino-resource-group-managers/src/test/java/io/trino/plugin/resourcegroups/db/TestDbResourceGroupsPostgresqlFlywayMigration.java @@ -14,7 +14,7 @@ package io.trino.plugin.resourcegroups.db; import org.testcontainers.containers.JdbcDatabaseContainer; -import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.postgresql.PostgreSQLContainer; public class TestDbResourceGroupsPostgresqlFlywayMigration extends BaseTestDbResourceGroupsFlywayMigration @@ -22,7 +22,7 @@ public class TestDbResourceGroupsPostgresqlFlywayMigration @Override protected final JdbcDatabaseContainer startContainer() { - JdbcDatabaseContainer container = new PostgreSQLContainer<>("postgres:11"); + JdbcDatabaseContainer container = new PostgreSQLContainer("postgres:11"); container.start(); return container; } diff --git a/plugin/trino-session-property-managers/pom.xml b/plugin/trino-session-property-managers/pom.xml index dd5847ba9cbe..9b15c2928847 100644 --- a/plugin/trino-session-property-managers/pom.xml +++ b/plugin/trino-session-property-managers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -216,14 +216,33 @@ org.testcontainers - mysql + testcontainers test org.testcontainers - testcontainers + testcontainers-mysql test + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + + + diff --git a/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/db/DbSessionPropertyManagerFactory.java b/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/db/DbSessionPropertyManagerFactory.java index 47069135b194..1ba0acfb189c 100644 --- a/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/db/DbSessionPropertyManagerFactory.java +++ b/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/db/DbSessionPropertyManagerFactory.java @@ -48,6 +48,7 @@ public SessionPropertyConfigurationManager create(Map config, Se Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/file/FileSessionPropertyManagerFactory.java b/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/file/FileSessionPropertyManagerFactory.java index 21b726dfc65d..5b7c8bf5185b 100644 --- a/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/file/FileSessionPropertyManagerFactory.java +++ b/plugin/trino-session-property-managers/src/main/java/io/trino/plugin/session/file/FileSessionPropertyManagerFactory.java @@ -41,6 +41,7 @@ public SessionPropertyConfigurationManager create(Map config, Se Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-session-property-managers/src/test/java/io/trino/plugin/session/db/TestingMySqlContainer.java b/plugin/trino-session-property-managers/src/test/java/io/trino/plugin/session/db/TestingMySqlContainer.java index 6f362e6a2f51..8944287a2e51 100644 --- a/plugin/trino-session-property-managers/src/test/java/io/trino/plugin/session/db/TestingMySqlContainer.java +++ b/plugin/trino-session-property-managers/src/test/java/io/trino/plugin/session/db/TestingMySqlContainer.java @@ -13,10 +13,10 @@ */ package io.trino.plugin.session.db; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; public class TestingMySqlContainer - extends MySQLContainer + extends MySQLContainer { public TestingMySqlContainer() { diff --git a/plugin/trino-singlestore/pom.xml b/plugin/trino-singlestore/pom.xml index 2772d4eee7bb..da636d0a9505 100644 --- a/plugin/trino-singlestore/pom.xml +++ b/plugin/trino-singlestore/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -174,6 +174,12 @@ test + + io.trino + trino-testing-containers + test + + io.trino trino-testing-services @@ -218,13 +224,13 @@ org.testcontainers - jdbc + testcontainers test org.testcontainers - testcontainers + testcontainers-jdbc test diff --git a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/SingleStoreQueryRunner.java b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/SingleStoreQueryRunner.java index e0bdbc692d56..f77625396ee7 100644 --- a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/SingleStoreQueryRunner.java +++ b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/SingleStoreQueryRunner.java @@ -85,7 +85,7 @@ public DistributedQueryRunner build() queryRunner.installPlugin(new SingleStorePlugin()); queryRunner.createCatalog("singlestore", "singlestore", connectorProperties); - queryRunner.execute("CREATE SCHEMA tpch"); + queryRunner.execute("CREATE SCHEMA IF NOT EXISTS tpch"); copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, initialTables); return queryRunner; diff --git a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestingSingleStoreServer.java b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestingSingleStoreServer.java index 017c4dcc229b..4f75955e2c49 100644 --- a/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestingSingleStoreServer.java +++ b/plugin/trino-singlestore/src/test/java/io/trino/plugin/singlestore/TestingSingleStoreServer.java @@ -17,12 +17,17 @@ import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.utility.DockerImageName; +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.Set; +import static io.trino.testing.containers.TestContainers.startOrReuse; + public class TestingSingleStoreServer extends JdbcDatabaseContainer { @@ -33,6 +38,8 @@ public class TestingSingleStoreServer public static final Integer SINGLESTORE_PORT = 3306; + private final Closeable cleanup; + public TestingSingleStoreServer() { this(DEFAULT_VERSION); @@ -43,7 +50,7 @@ public TestingSingleStoreServer(String version) super(DockerImageName.parse(DEFAULT_TAG)); addEnv("ROOT_PASSWORD", "memsql_root_password"); addEnv("SINGLESTORE_VERSION", version); - start(); + cleanup = startOrReuse(this); } @Override @@ -89,6 +96,17 @@ public String getTestQueryString() return "SELECT 1"; } + @Override + public void close() + { + try { + cleanup.close(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public void execute(String sql) { execute(sql, getUsername(), getPassword()); diff --git a/plugin/trino-snowflake/pom.xml b/plugin/trino-snowflake/pom.xml index 5f9db5218255..9e19c7057b80 100644 --- a/plugin/trino-snowflake/pom.xml +++ b/plugin/trino-snowflake/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-spooling-filesystem/pom.xml b/plugin/trino-spooling-filesystem/pom.xml index eba68c249989..c269be5c3e98 100644 --- a/plugin/trino-spooling-filesystem/pom.xml +++ b/plugin/trino-spooling-filesystem/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -209,19 +209,19 @@ org.testcontainers - junit-jupiter + testcontainers test org.testcontainers - localstack + testcontainers-junit-jupiter test org.testcontainers - testcontainers + testcontainers-localstack test diff --git a/plugin/trino-spooling-filesystem/src/main/java/io/trino/spooling/filesystem/FileSystemSpoolingManagerFactory.java b/plugin/trino-spooling-filesystem/src/main/java/io/trino/spooling/filesystem/FileSystemSpoolingManagerFactory.java index 17097b36d702..492d6cdd00cb 100644 --- a/plugin/trino-spooling-filesystem/src/main/java/io/trino/spooling/filesystem/FileSystemSpoolingManagerFactory.java +++ b/plugin/trino-spooling-filesystem/src/main/java/io/trino/spooling/filesystem/FileSystemSpoolingManagerFactory.java @@ -53,6 +53,7 @@ public SpoolingManager create(Map config, SpoolingManagerContext Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-spooling-filesystem/src/test/java/io/trino/spooling/filesystem/AbstractFileSystemSegmentPrunerTest.java b/plugin/trino-spooling-filesystem/src/test/java/io/trino/spooling/filesystem/AbstractFileSystemSegmentPrunerTest.java index e45c61eca8f9..9bc70fdf9f6e 100644 --- a/plugin/trino-spooling-filesystem/src/test/java/io/trino/spooling/filesystem/AbstractFileSystemSegmentPrunerTest.java +++ b/plugin/trino-spooling-filesystem/src/test/java/io/trino/spooling/filesystem/AbstractFileSystemSegmentPrunerTest.java @@ -149,7 +149,7 @@ private Location writeDataSegment(TrinoFileSystem fileSystem, QueryId queryId, I FileSystemSpooledSegmentHandle handle = FileSystemSpooledSegmentHandle.random(ThreadLocalRandom.current(), "nodeId", context, ttl); Location location = layout().location(TEST_LOCATION, handle); try (OutputStream stream = fileSystem.newOutputFile(location).create()) { - stream.write(queryId.toString().getBytes(UTF_8)); + stream.write(queryId.id().getBytes(UTF_8)); return location; } catch (IOException e) { @@ -166,7 +166,7 @@ private List listFiles(TrinoFileSystem fileSystem, QueryId queryId) while (iterator.hasNext()) { FileEntry entry = iterator.next(); try (TrinoInputStream stream = fileSystem.newInputFile(entry.location()).newStream()) { - if (Arrays.equals(stream.readAllBytes(), queryId.toString().getBytes(UTF_8))) { + if (Arrays.equals(stream.readAllBytes(), queryId.id().getBytes(UTF_8))) { files.add(entry.location()); } } diff --git a/plugin/trino-sqlserver/pom.xml b/plugin/trino-sqlserver/pom.xml index 8f93b449cca2..da1f51b4713a 100644 --- a/plugin/trino-sqlserver/pom.xml +++ b/plugin/trino-sqlserver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -245,19 +245,19 @@ org.testcontainers - jdbc + testcontainers test org.testcontainers - mssqlserver + testcontainers-jdbc test org.testcontainers - testcontainers + testcontainers-mssqlserver test diff --git a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java index e45cb0e75ef6..5e935eb13a15 100644 --- a/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java +++ b/plugin/trino-sqlserver/src/main/java/io/trino/plugin/sqlserver/SqlServerClient.java @@ -394,7 +394,7 @@ public Collection listSchemas(Connection connection) return schemaNames.build(); } catch (SQLException e) { - throw new RuntimeException(e); + throw new TrinoException(JDBC_ERROR, e); } } diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java index 7b5eda22dd26..ed9561abf7bd 100644 --- a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/BaseSqlServerConnectorTest.java @@ -223,6 +223,15 @@ public void testPredicatePushdown() .returnsEmptyResult() .isFullyPushedDown(); + // varchar predicate over join + Session joinPushdownEnabled = joinPushdownEnabled(getSession()); + assertThat(query(joinPushdownEnabled, "SELECT c.name, n.name FROM customer c JOIN nation n ON c.custkey = n.nationkey WHERE n.name = 'POLAND'")) + .isFullyPushedDown(); + + // join on varchar columns + assertThat(query(joinPushdownEnabled, "SELECT n.name, n2.regionkey FROM nation n JOIN nation n2 ON n.name = n2.name")) + .isFullyPushedDown(); + // bigint equality assertThat(query("SELECT regionkey, nationkey, name FROM nation WHERE nationkey = 19")) .matches("VALUES (BIGINT '3', BIGINT '19', CAST('ROMANIA' AS varchar(25)))") @@ -286,15 +295,6 @@ public void testPredicatePushdown() assertThat(query("SELECT * FROM " + testTable.getName() + " WHERE long_decimal = 123456789.987654321")) .matches("VALUES (CAST(123.321 AS decimal(9,3)), CAST(123456789.987654321 AS decimal(30, 10)))") .isFullyPushedDown(); - - // varchar predicate over join - Session joinPushdownEnabled = joinPushdownEnabled(getSession()); - assertThat(query(joinPushdownEnabled, "SELECT c.name, n.name FROM customer c JOIN nation n ON c.custkey = n.nationkey WHERE n.name = 'POLAND'")) - .isFullyPushedDown(); - - // join on varchar columns - assertThat(query(joinPushdownEnabled, "SELECT n.name, n2.regionkey FROM nation n JOIN nation n2 ON n.name = n2.name")) - .isFullyPushedDown(); } } diff --git a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java index 099e0b46091a..0c6d914aba69 100644 --- a/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java +++ b/plugin/trino-sqlserver/src/test/java/io/trino/plugin/sqlserver/TestingSqlServer.java @@ -18,8 +18,8 @@ import dev.failsafe.Timeout; import io.airlift.log.Logger; import io.trino.testing.sql.SqlExecutor; -import org.testcontainers.containers.MSSQLServerContainer; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.mssqlserver.MSSQLServerContainer; import org.testcontainers.utility.DockerImageName; import java.io.Closeable; @@ -65,7 +65,7 @@ public final class TestingSqlServer private static final DockerImageName IMAGE_NAME = DockerImageName.parse("mcr.microsoft.com/mssql/server"); public static final String LATEST_VERSION = "2019-CU28-ubuntu-20.04"; - private final MSSQLServerContainer container; + private final MSSQLServerContainer container; private final String databaseName; private final Closeable cleanup; @@ -102,7 +102,7 @@ public TestingSqlServer(String version, BiConsumer database private static InitializedState createContainer(String version, BiConsumer databaseSetUp) { class TestingMSSQLServerContainer - extends MSSQLServerContainer + extends MSSQLServerContainer { TestingMSSQLServerContainer(DockerImageName dockerImageName) { @@ -130,7 +130,7 @@ public String getUsername() } String databaseName = "database_" + UUID.randomUUID().toString().replace("-", ""); - MSSQLServerContainer container = new TestingMSSQLServerContainer(IMAGE_NAME.withTag(version)); + MSSQLServerContainer container = new TestingMSSQLServerContainer(IMAGE_NAME.withTag(version)); container.acceptLicense(); // enable case sensitive (see the CS below) collation for SQL identifiers container.addEnv("MSSQL_COLLATION", "Latin1_General_CS_AS"); @@ -174,7 +174,7 @@ public void execute(String sql) sqlExecutorForContainer(container).execute(sql); } - private static SqlExecutor sqlExecutorForContainer(MSSQLServerContainer container) + private static SqlExecutor sqlExecutorForContainer(MSSQLServerContainer container) { requireNonNull(container, "container is null"); return sql -> { @@ -222,11 +222,11 @@ public void close() private static class InitializedState { - private final MSSQLServerContainer container; + private final MSSQLServerContainer container; private final String databaseName; private final Closeable cleanup; - public InitializedState(MSSQLServerContainer container, String databaseName, Closeable cleanup) + public InitializedState(MSSQLServerContainer container, String databaseName, Closeable cleanup) { this.container = container; this.databaseName = databaseName; diff --git a/plugin/trino-teradata-functions/pom.xml b/plugin/trino-teradata-functions/pom.xml index 3f248512c08e..3aa0b9a14b96 100644 --- a/plugin/trino-teradata-functions/pom.xml +++ b/plugin/trino-teradata-functions/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-teradata/README.md b/plugin/trino-teradata/README.md new file mode 100644 index 000000000000..ee45e8f3b908 --- /dev/null +++ b/plugin/trino-teradata/README.md @@ -0,0 +1,42 @@ +# Teradata Connector Developer Notes + +The Teradata connector module has both unit tests and integration tests. +The integration tests require access to a [Teradata ClearScape Analytics™ Experience](https://clearscape.teradata.com/sign-in). +You can follow the steps below to run the integration tests locally. + +## Prerequisites + +#### 1. Create a new ClearScape Analytics™ Experience account + +If you don't already have one, sign up at: + +[Teradata ClearScape Analytics™ Experience](https://www.teradata.com/getting-started/demos/clearscape-analytics) + +#### 2. Login + +Sign in with your new account at: + +[ClearScape Analytics™ Experience Login](https://clearscape.teradata.com/sign-in) + +#### 3. Collect the API Token + +Use the **Copy API Token** button in the UI to retrieve your token. + +#### 4. Define the following environment variables + +⚠️ **Note:** The Teradata database password must be **at least 8 characters long**. + +``` +export CLEARSCAPE_TOKEN= +export CLEARSCAPE_PASSWORD= +``` + +## Running Integration Tests + +Once the environment variables are set, run the integration tests with: + +⚠️ **Note:** Run the following command from the Trino parent directory. + + ``` + ./mvnw clean install -pl :trino-teradata +``` diff --git a/plugin/trino-teradata/pom.xml b/plugin/trino-teradata/pom.xml new file mode 100644 index 000000000000..242838552093 --- /dev/null +++ b/plugin/trino-teradata/pom.xml @@ -0,0 +1,282 @@ + + + 4.0.0 + + io.trino + trino-root + 479-SNAPSHOT + ../../pom.xml + + + trino-teradata + trino-plugin + ${project.artifactId} + Trino - Teradata connector + + + true + + + + + + com.google.guava + guava + + + + com.google.inject + guice + classes + + + + io.airlift + configuration + + + + io.airlift + log + + + + io.trino + trino-base-jdbc + + + + io.trino + trino-plugin-toolkit + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.airlift + slice + provided + + + + io.opentelemetry + opentelemetry-api + provided + + + + io.opentelemetry + opentelemetry-api-incubator + provided + + + + io.opentelemetry + opentelemetry-context + provided + + + + io.trino + trino-spi + provided + + + + org.openjdk.jol + jol-core + provided + + + + com.google.errorprone + error_prone_annotations + runtime + + + + com.teradata.jdbc + terajdbc + 20.00.00.49 + runtime + + + + io.airlift + concurrent + runtime + + + + io.airlift + json + runtime + + + + io.airlift + log-manager + runtime + + + + io.airlift + units + runtime + + + + io.airlift + configuration-testing + test + + + + io.airlift + junit-extensions + test + + + + io.airlift + testing + test + + + + io.airlift + tracing + test + + + + io.trino + trino-base-jdbc + test-jar + test + + + + io.trino + trino-exchange-filesystem + test + + + + io.trino + trino-exchange-filesystem + test-jar + test + + + + io.trino + trino-jmx + test + + + + io.trino + trino-main + test + + + + io.trino + trino-main + test-jar + test + + + + io.trino + trino-parser + test + + + + io.trino + trino-plugin-toolkit + test-jar + test + + + + io.trino + trino-testing + test + + + + io.trino + trino-testing-containers + test + + + + io.trino + trino-testing-services + test + + + + io.trino + trino-tpch + test + + + + io.trino.tpch + tpch + test + + + + org.assertj + assertj-core + test + + + + org.jetbrains + annotations + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + 3.8.1 + + + com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-databind + com.google.guava:guava + io.airlift:log + + + + + + + diff --git a/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/LogonMechanism.java b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/LogonMechanism.java new file mode 100644 index 000000000000..18358011ba7e --- /dev/null +++ b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/LogonMechanism.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.teradata; + +public enum LogonMechanism +{ + TD2("TD2"); + + private final String mechanism; + + LogonMechanism(String mechanism) + { + this.mechanism = mechanism; + } + + public static LogonMechanism fromString(String value) + { + for (LogonMechanism logonMechanism : values()) { + if (logonMechanism.getMechanism().equalsIgnoreCase(value)) { + return logonMechanism; + } + } + throw new IllegalArgumentException("Unknown logon mechanism: " + value); + } + + public String getMechanism() + { + return mechanism; + } +} diff --git a/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataClient.java b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataClient.java new file mode 100644 index 000000000000..8d5bacc685fd --- /dev/null +++ b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataClient.java @@ -0,0 +1,382 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.teradata; + +import com.google.inject.Inject; +import io.trino.plugin.base.mapping.IdentifierMapping; +import io.trino.plugin.jdbc.BaseJdbcClient; +import io.trino.plugin.jdbc.BaseJdbcConfig; +import io.trino.plugin.jdbc.CaseSensitivity; +import io.trino.plugin.jdbc.ColumnMapping; +import io.trino.plugin.jdbc.ConnectionFactory; +import io.trino.plugin.jdbc.JdbcColumnHandle; +import io.trino.plugin.jdbc.JdbcOutputTableHandle; +import io.trino.plugin.jdbc.JdbcTableHandle; +import io.trino.plugin.jdbc.JdbcTypeHandle; +import io.trino.plugin.jdbc.PredicatePushdownController; +import io.trino.plugin.jdbc.QueryBuilder; +import io.trino.plugin.jdbc.RemoteTableName; +import io.trino.plugin.jdbc.WriteMapping; +import io.trino.plugin.jdbc.logging.RemoteQueryModifier; +import io.trino.spi.TrinoException; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ColumnPosition; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.type.CharType; +import io.trino.spi.type.DecimalType; +import io.trino.spi.type.Decimals; +import io.trino.spi.type.Type; +import io.trino.spi.type.VarcharType; +import org.weakref.jmx.$internal.guava.collect.ImmutableMap; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Types; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.plugin.jdbc.CaseSensitivity.CASE_INSENSITIVE; +import static io.trino.plugin.jdbc.CaseSensitivity.CASE_SENSITIVE; +import static io.trino.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; +import static io.trino.plugin.jdbc.PredicatePushdownController.CASE_INSENSITIVE_CHARACTER_PUSHDOWN; +import static io.trino.plugin.jdbc.PredicatePushdownController.FULL_PUSHDOWN; +import static io.trino.plugin.jdbc.StandardColumnMappings.bigintColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.bigintWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.charReadFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.charWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.dateColumnMappingUsingLocalDate; +import static io.trino.plugin.jdbc.StandardColumnMappings.dateWriteFunctionUsingLocalDate; +import static io.trino.plugin.jdbc.StandardColumnMappings.decimalColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.doubleColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.doubleWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.integerColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.integerWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.realWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.smallintColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.smallintWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.tinyintColumnMapping; +import static io.trino.plugin.jdbc.StandardColumnMappings.tinyintWriteFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.varcharReadFunction; +import static io.trino.plugin.jdbc.StandardColumnMappings.varcharWriteFunction; +import static io.trino.plugin.jdbc.TypeHandlingJdbcSessionProperties.getUnsupportedTypeHandling; +import static io.trino.plugin.jdbc.UnsupportedTypeHandling.CONVERT_TO_VARCHAR; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.CharType.createCharType; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.RealType.REAL; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.VarcharType.createUnboundedVarcharType; +import static io.trino.spi.type.VarcharType.createVarcharType; +import static java.lang.String.format; + +public class TeradataClient + extends BaseJdbcClient +{ + private static final PredicatePushdownController TERADATA_STRING_PUSHDOWN = FULL_PUSHDOWN; + private final TeradataConfig.TeradataCaseSensitivity teradataJDBCCaseSensitivity; + + @Inject + public TeradataClient( + BaseJdbcConfig config, + TeradataConfig teradataConfig, + ConnectionFactory connectionFactory, + QueryBuilder queryBuilder, + IdentifierMapping identifierMapping, + RemoteQueryModifier remoteQueryModifier) + { + super("\"", connectionFactory, queryBuilder, config.getJdbcTypesMappedToVarchar(), identifierMapping, remoteQueryModifier, true); + this.teradataJDBCCaseSensitivity = teradataConfig.getTeradataCaseSensitivity(); + } + + private static ColumnMapping charColumnMapping(int charLength, boolean isCaseSensitive) + { + if (charLength > CharType.MAX_LENGTH) { + return varcharColumnMapping(charLength, isCaseSensitive); + } + CharType charType = createCharType(charLength); + return ColumnMapping.sliceMapping( + charType, + charReadFunction(charType), + charWriteFunction(), + isCaseSensitive ? TERADATA_STRING_PUSHDOWN : CASE_INSENSITIVE_CHARACTER_PUSHDOWN); + } + + private static ColumnMapping varcharColumnMapping(int varcharLength, boolean isCaseSensitive) + { + VarcharType varcharType = varcharLength <= VarcharType.MAX_LENGTH + ? createVarcharType(varcharLength) + : createUnboundedVarcharType(); + return ColumnMapping.sliceMapping( + varcharType, + varcharReadFunction(varcharType), + varcharWriteFunction(), + isCaseSensitive ? TERADATA_STRING_PUSHDOWN : CASE_INSENSITIVE_CHARACTER_PUSHDOWN); + } + + private boolean deriveCaseSensitivity(CaseSensitivity caseSensitivity) + { + return switch (teradataJDBCCaseSensitivity) { + case CASE_INSENSITIVE -> false; + case CASE_SENSITIVE -> true; + default -> caseSensitivity != null; + }; + } + + @Override + protected void createSchema(ConnectorSession session, Connection connection, String remoteSchemaName) + { + execute(session, format( + "CREATE DATABASE %s AS PERMANENT = 60000000, SPOOL = 120000000", + quoted(remoteSchemaName))); + } + + @Override + protected void copyTableSchema(ConnectorSession session, Connection connection, String catalogName, String schemaName, String tableName, String newTableName, + List columnNames) + { + String tableCopyFormat = "CREATE TABLE %s AS ( SELECT * FROM %s ) WITH DATA"; + String sql = format( + tableCopyFormat, + quoted(catalogName, schemaName, newTableName), + quoted(catalogName, schemaName, tableName)); + try { + execute(session, connection, sql); + } + catch (SQLException e) { + throw new TrinoException(JDBC_ERROR, e); + } + } + + @Override + protected void verifySchemaName(DatabaseMetaData databaseMetadata, String schemaName) + throws SQLException + { + int schemaNameLimit = databaseMetadata.getMaxSchemaNameLength(); + if (schemaName.length() > schemaNameLimit) { + throw new TrinoException(NOT_SUPPORTED, format("Schema name must be shorter than or equal to '%s' characters but got '%s'", schemaNameLimit, schemaName.length())); + } + } + + @Override + protected void verifyTableName(DatabaseMetaData databaseMetadata, String tableName) + throws SQLException + { + if (tableName.length() > databaseMetadata.getMaxTableNameLength()) { + throw new TrinoException(NOT_SUPPORTED, format("Table name must be shorter than or equal to '%s' characters but got '%s'", databaseMetadata.getMaxTableNameLength(), + tableName.length())); + } + } + + @Override + protected void verifyColumnName(DatabaseMetaData databaseMetadata, String columnName) + throws SQLException + { + if (columnName.length() > databaseMetadata.getMaxColumnNameLength()) { + throw new TrinoException(NOT_SUPPORTED, format("Column name must be shorter than or equal to '%s' characters but got '%s': '%s'", + databaseMetadata.getMaxColumnNameLength(), columnName.length(), columnName)); + } + } + + @Override + protected void dropSchema(ConnectorSession session, Connection connection, String remoteSchemaName, boolean cascade) + throws SQLException + { + if (cascade) { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support dropping schemas with CASCADE option"); + } + String dropSchema = "DROP DATABASE " + quoted(remoteSchemaName); + execute(session, connection, dropSchema); + } + + @Override + public void renameSchema(ConnectorSession session, String schemaName, String newSchemaName) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support renaming schema"); + } + + @Override + public OptionalLong delete(ConnectorSession session, JdbcTableHandle handle) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support modifying table rows"); + } + + @Override + public void truncateTable(ConnectorSession session, JdbcTableHandle handle) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support truncating tables"); + } + + @Override + public void dropColumn(ConnectorSession session, JdbcTableHandle handle, JdbcColumnHandle column) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support dropping columns"); + } + + @Override + public void renameColumn(ConnectorSession session, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support renaming columns"); + } + + @Override + public void renameTable(ConnectorSession session, JdbcTableHandle handle, SchemaTableName newTableName) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support renaming tables"); + } + + @Override + public JdbcOutputTableHandle beginInsertTable(ConnectorSession session, JdbcTableHandle tableHandle, List columns) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support inserts"); + } + + @Override + public void setColumnType(ConnectorSession session, JdbcTableHandle handle, JdbcColumnHandle column, Type type) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support setting column types"); + } + + @Override + public void addColumn(ConnectorSession session, JdbcTableHandle handle, ColumnMetadata column, ColumnPosition position) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support add column operations"); + } + + @Override + public void dropNotNullConstraint(ConnectorSession session, JdbcTableHandle handle, JdbcColumnHandle column) + { + throw new TrinoException(NOT_SUPPORTED, "This connector does not support dropping a not null constraint"); + } + + @Override + protected Map getCaseSensitivityForColumns(ConnectorSession session, Connection connection, SchemaTableName schemaTableName, + RemoteTableName remoteTableName) + { + // try to use result set metadata from select * from table to populate the mapping + try { + HashMap caseMap = new HashMap<>(); + String sql = format("select * from %s.%s where 0=1", schemaTableName.getSchemaName(), schemaTableName.getTableName()); + PreparedStatement pstmt = connection.prepareStatement(sql); + ResultSetMetaData rsmd = pstmt.getMetaData(); + int columnCount = rsmd.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + caseMap.put(rsmd.getColumnName(i), rsmd.isCaseSensitive(i) ? CASE_SENSITIVE : CASE_INSENSITIVE); + } + pstmt.close(); + return caseMap; + } + catch (SQLException e) { + // behavior of base jdbc + return ImmutableMap.of(); + } + } + + @Override + public Optional toColumnMapping(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle) + { + // this method should ultimately encompass all the expected teradata data types + Optional mapping = getForcedMappingToVarchar(typeHandle); + if (mapping.isPresent()) { + return mapping; + } + + switch (typeHandle.jdbcType()) { + case Types.TINYINT: + return Optional.of(tinyintColumnMapping()); + case Types.SMALLINT: + return Optional.of(smallintColumnMapping()); + case Types.INTEGER: + return Optional.of(integerColumnMapping()); + case Types.BIGINT: + return Optional.of(bigintColumnMapping()); + case Types.REAL: + case Types.DOUBLE: + case Types.FLOAT: + // teradata float is 64 bit + // trino double is 64 bit + // teradata float / real / double precision all map to jdbc type float + return Optional.of(doubleColumnMapping()); + case Types.NUMERIC: + case Types.DECIMAL: + return numberMapping(typeHandle); + case Types.CHAR: + return Optional.of(charColumnMapping(typeHandle.requiredColumnSize(), deriveCaseSensitivity(typeHandle.caseSensitivity().orElse(null)))); + case Types.VARCHAR: + // see prior note on trino case sensitivity + return Optional.of(varcharColumnMapping(typeHandle.requiredColumnSize(), deriveCaseSensitivity(typeHandle.caseSensitivity().orElse(null)))); + case Types.DATE: + return Optional.of(dateColumnMappingUsingLocalDate()); + } + + if (getUnsupportedTypeHandling(session) == CONVERT_TO_VARCHAR) { + return mapToUnboundedVarchar(typeHandle); + } + + return Optional.empty(); + } + + private Optional numberMapping(JdbcTypeHandle typeHandle) + { + int precision = typeHandle.requiredColumnSize(); + int scale = typeHandle.requiredDecimalDigits(); + if (precision > Decimals.MAX_PRECISION) { + // this will trigger for number(*) as precision is 40 + return Optional.of(decimalColumnMapping(createDecimalType(Decimals.MAX_PRECISION, scale))); + } + return Optional.of(decimalColumnMapping(createDecimalType(precision, scale))); + } + + @Override + public WriteMapping toWriteMapping(ConnectorSession session, Type type) + { + return switch (type) { + case Type typeInstance when typeInstance == TINYINT -> WriteMapping.longMapping("smallint", tinyintWriteFunction()); + case Type typeInstance when typeInstance == SMALLINT -> WriteMapping.longMapping("smallint", smallintWriteFunction()); + case Type typeInstance when typeInstance == INTEGER -> WriteMapping.longMapping("integer", integerWriteFunction()); + case Type typeInstance when typeInstance == BIGINT -> WriteMapping.longMapping("bigint", bigintWriteFunction()); + case Type typeInstance when typeInstance == REAL -> WriteMapping.longMapping("FLOAT", realWriteFunction()); + case Type typeInstance when typeInstance == DOUBLE -> WriteMapping.doubleMapping("double precision", doubleWriteFunction()); + case Type typeInstance when typeInstance == DATE -> WriteMapping.longMapping("date", dateWriteFunctionUsingLocalDate()); + case DecimalType decimalTypeInstance -> { + String dataType = String.format("decimal(%s, %s)", decimalTypeInstance.getPrecision(), decimalTypeInstance.getScale()); + if (decimalTypeInstance.isShort()) { + yield WriteMapping.longMapping(dataType, shortDecimalWriteFunction(decimalTypeInstance)); + } + yield WriteMapping.objectMapping(dataType, longDecimalWriteFunction(decimalTypeInstance)); + } + case CharType charTypeInstance -> WriteMapping.sliceMapping("char(" + charTypeInstance.getLength() + ")", charWriteFunction()); + case VarcharType varcharTypeInstance -> { + String dataType = varcharTypeInstance.isUnbounded() + ? "clob" + : "varchar(" + varcharTypeInstance.getBoundedLength() + ")"; + yield WriteMapping.sliceMapping(dataType, varcharWriteFunction()); + } + default -> throw new TrinoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName()); + }; + } +} diff --git a/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataClientModule.java b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataClientModule.java new file mode 100644 index 000000000000..80701879c317 --- /dev/null +++ b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataClientModule.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.teradata; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.opentelemetry.api.OpenTelemetry; +import io.trino.plugin.jdbc.BaseJdbcConfig; +import io.trino.plugin.jdbc.ConnectionFactory; +import io.trino.plugin.jdbc.DriverConnectionFactory; +import io.trino.plugin.jdbc.ForBaseJdbc; +import io.trino.plugin.jdbc.JdbcClient; +import io.trino.plugin.jdbc.JdbcJoinPushdownSupportModule; +import io.trino.plugin.jdbc.JdbcStatisticsConfig; +import io.trino.plugin.jdbc.credential.CredentialProvider; + +import java.sql.Driver; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +import static io.airlift.configuration.ConfigBinder.configBinder; + +public class TeradataClientModule + extends AbstractConfigurationAwareModule +{ + @Provides + @Singleton + @ForBaseJdbc + public static ConnectionFactory getConnectionFactory(BaseJdbcConfig config, TeradataConfig teradataConfig, CredentialProvider credentialProvider, OpenTelemetry openTelemetry) + throws SQLException + { + Properties connectionProperties = new Properties(); + Driver driver = DriverManager.getDriver(config.getConnectionUrl()); + String logonMechanism = LogonMechanism.fromString(teradataConfig.getLogMech()).getMechanism(); + connectionProperties.put("LOGMECH", logonMechanism); + if (!logonMechanism.equals("TD2")) { + throw new IllegalArgumentException("Unsupported logon mechanism: " + logonMechanism); + } + return DriverConnectionFactory.builder(driver, config.getConnectionUrl(), credentialProvider).setConnectionProperties(connectionProperties).setOpenTelemetry(openTelemetry).build(); + } + + @Override + public void setup(Binder binder) + { + configBinder(binder).bindConfig(TeradataConfig.class); + binder.bind(JdbcClient.class).annotatedWith(ForBaseJdbc.class).to(TeradataClient.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(JdbcStatisticsConfig.class); + install(new JdbcJoinPushdownSupportModule()); + } +} diff --git a/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataConfig.java b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataConfig.java new file mode 100644 index 000000000000..e4ede4a6461a --- /dev/null +++ b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataConfig.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.teradata; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.trino.plugin.jdbc.BaseJdbcConfig; + +public class TeradataConfig + extends BaseJdbcConfig +{ + private String logMech = "TD2"; + private TeradataCaseSensitivity teradataCaseSensitivity = TeradataCaseSensitivity.CASE_SENSITIVE; + + public String getLogMech() + { + return logMech; + } + + @Config("logon-mechanism") + @ConfigDescription("Specifies the logon mechanism for Teradata (default: TD2). Use 'TD2' for TD2 authentication.") + public TeradataConfig setLogMech(String logMech) + { + this.logMech = logMech; + return this; + } + + public TeradataCaseSensitivity getTeradataCaseSensitivity() + { + return teradataCaseSensitivity; + } + + @Config("teradata.case-sensitivity") + @ConfigDescription("How char/varchar columns' case sensitivity will be exposed to Trino (default: CASE_SENSITIVE). Possible values: CASE_INSENSITIVE, CASE_SENSITIVE, AS_DEFINED.") + public TeradataConfig setTeradataCaseSensitivity(TeradataCaseSensitivity teradataCaseSensitivity) + { + this.teradataCaseSensitivity = teradataCaseSensitivity; + return this; + } + + public enum TeradataCaseSensitivity + { + CASE_INSENSITIVE, CASE_SENSITIVE, AS_DEFINED + } +} diff --git a/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataPlugin.java b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataPlugin.java new file mode 100644 index 000000000000..d11110edfbed --- /dev/null +++ b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/TeradataPlugin.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.teradata; + +import io.trino.plugin.jdbc.JdbcPlugin; + +public class TeradataPlugin + extends JdbcPlugin +{ + public TeradataPlugin() + { + super("teradata", TeradataClientModule::new); + } +} diff --git a/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/util/TeradataConstants.java b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/util/TeradataConstants.java new file mode 100644 index 000000000000..ea406c58cb06 --- /dev/null +++ b/plugin/trino-teradata/src/main/java/io/trino/plugin/teradata/util/TeradataConstants.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.teradata.util; + +public interface TeradataConstants +{ + int TERADATA_OBJECT_NAME_LIMIT = 128; +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/AuthenticationConfig.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/AuthenticationConfig.java new file mode 100644 index 000000000000..e4dad3ed36ea --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/AuthenticationConfig.java @@ -0,0 +1,24 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration; + +public record AuthenticationConfig( + String userName, + String password) +{ + public AuthenticationConfig() + { + this(null, null); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/DatabaseConfig.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/DatabaseConfig.java new file mode 100644 index 000000000000..a0f3e72f3fd2 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/DatabaseConfig.java @@ -0,0 +1,173 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration; + +import io.trino.plugin.teradata.LogonMechanism; + +import java.util.Map; + +public class DatabaseConfig +{ + private final String jdbcUrl; + private final String hostName; + private final String databaseName; + private final boolean useClearScape; + private final LogonMechanism logMech; + private final AuthenticationConfig authConfig; + private final String clearScapeEnvName; + private final Map jdbcProperties; + + private DatabaseConfig(Builder builder) + { + this.jdbcUrl = builder.jdbcUrl; + this.hostName = builder.hostName; + this.databaseName = builder.databaseName; + this.useClearScape = builder.useClearScape; + this.logMech = builder.logMech; + this.authConfig = builder.authConfig; + this.clearScapeEnvName = builder.clearScapeEnvName; + this.jdbcProperties = builder.jdbcProperties; + } + + public static Builder builder() + { + return new Builder(); + } + + public Builder toBuilder() + { + return builder() + .jdbcUrl(this.jdbcUrl) + .hostName(this.hostName) + .databaseName(this.databaseName) + .useClearScape(this.useClearScape) + .logMech(this.logMech) + .authConfig(this.authConfig) + .clearScapeEnvName(this.clearScapeEnvName) + .jdbcProperties(this.jdbcProperties); + } + + public String getJdbcUrl() + { + return jdbcUrl; + } + + public String getDatabaseName() + { + return databaseName; + } + + public boolean isUseClearScape() + { + return useClearScape; + } + + public LogonMechanism getLogMech() + { + return logMech; + } + + public AuthenticationConfig getAuthConfig() + { + return authConfig; + } + + public String getClearScapeEnvName() + { + return clearScapeEnvName; + } + + public Map getJdbcProperties() + { + return jdbcProperties; + } + + public String getHostName() + { + return hostName; + } + + public String getTMode() + { + if (jdbcProperties != null && jdbcProperties.containsKey("TMODE")) { + return jdbcProperties.get("TMODE"); + } + return "ANSI"; + } + + public static class Builder + { + private String jdbcUrl; + private String hostName; + private String databaseName = "trino"; + private boolean useClearScape; + private LogonMechanism logMech = LogonMechanism.TD2; + private AuthenticationConfig authConfig = new AuthenticationConfig(); + private String clearScapeEnvName; + private Map jdbcProperties; + + public Builder jdbcUrl(String jdbcUrl) + { + this.jdbcUrl = jdbcUrl; + return this; + } + + public Builder databaseName(String databaseName) + { + this.databaseName = databaseName; + return this; + } + + public Builder useClearScape(boolean useClearScape) + { + this.useClearScape = useClearScape; + return this; + } + + public Builder logMech(LogonMechanism logMech) + { + this.logMech = logMech; + return this; + } + + public Builder authConfig(AuthenticationConfig authConfig) + { + this.authConfig = authConfig; + return this; + } + + public Builder clearScapeEnvName(String clearScapeEnvName) + { + this.clearScapeEnvName = clearScapeEnvName; + return this; + } + + public Builder jdbcProperties(Map jdbcProperties) + { + this.jdbcProperties = jdbcProperties; + return this; + } + + public Builder hostName(String hostName) + { + this.hostName = hostName; + return this; + } + + public DatabaseConfig build() + { + return new DatabaseConfig(this); + } + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/DatabaseConfigFactory.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/DatabaseConfigFactory.java new file mode 100644 index 000000000000..a5139802eaeb --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/DatabaseConfigFactory.java @@ -0,0 +1,82 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration; + +import io.trino.plugin.integration.util.TeradataTestConstants; +import io.trino.plugin.teradata.LogonMechanism; + +import java.util.HashMap; +import java.util.Map; + +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; + +public class DatabaseConfigFactory +{ + private static final String DEFAULT_LOG_MECH = "TD2"; + + private DatabaseConfigFactory() {} + + public static DatabaseConfig create(String envName) + { + String userName = null; + String password = null; + String hostName = null; + + if (!isEnvSet("CLEARSCAPE_TOKEN")) { + hostName = requireEnv("hostname"); + } + + String logMech = DEFAULT_LOG_MECH; + if (isEnvSet("logMech")) { + logMech = requireEnv("logMech"); + } + if (DEFAULT_LOG_MECH.equals(logMech)) { + if (isEnvSet("CLEARSCAPE_TOKEN")) { + userName = TeradataTestConstants.ENV_CLEARSCAPE_USERNAME; + password = requireEnv("CLEARSCAPE_PASSWORD"); + } + else { + userName = requireEnv("username"); + password = requireEnv("password"); + } + } + LogonMechanism logonMechanism = LogonMechanism.fromString(logMech); + String databaseName = envName.replace("-", "_"); + + AuthenticationConfig authConfig = createAuthConfig(userName, password); + return DatabaseConfig.builder() + .hostName(hostName) + .databaseName(databaseName) + .useClearScape(isEnvSet("CLEARSCAPE_TOKEN")) + .logMech(logonMechanism) + .authConfig(authConfig) + .clearScapeEnvName(envName) + .jdbcProperties(getJdbcProperties()) + .build(); + } + + public static Map getJdbcProperties() + { + Map propsMap = new HashMap<>(); + propsMap.put("TMODE", "ANSI"); + propsMap.put("CHARSET", "UTF8"); + return propsMap; + } + + private static AuthenticationConfig createAuthConfig(String username, String password) + { + return new AuthenticationConfig(username, password); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TeradataConnectorTest.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TeradataConnectorTest.java new file mode 100644 index 000000000000..8563edf49ca2 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TeradataConnectorTest.java @@ -0,0 +1,478 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration; + +import io.trino.Session; +import io.trino.plugin.integration.clearscape.ClearScapeEnvironmentUtils; +import io.trino.plugin.jdbc.BaseJdbcConnectorTest; +import io.trino.sql.query.QueryAssertions; +import io.trino.testing.QueryRunner; +import io.trino.testing.TestingConnectorBehavior; +import io.trino.testing.TestingNames; +import io.trino.testing.assertions.TrinoExceptionAssert; +import io.trino.testing.sql.SqlExecutor; +import io.trino.testing.sql.TestTable; +import org.assertj.core.api.AssertProvider; +import org.intellij.lang.annotations.Language; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.Consumer; + +import static io.trino.plugin.teradata.util.TeradataConstants.TERADATA_OBJECT_NAME_LIMIT; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.abort; + +final class TeradataConnectorTest + extends BaseJdbcConnectorTest +{ + private TestingTeradataServer database; + + private static void verifyResultOrFailure(AssertProvider queryAssertProvider, Consumer verifyResults, + Consumer verifyFailure) + { + requireNonNull(verifyResults, "verifyResults is null"); + requireNonNull(verifyFailure, "verifyFailure is null"); + QueryAssertions.QueryAssert queryAssert = assertThat(queryAssertProvider); + verifyResults.accept(queryAssert); + } + + @Override + protected SqlExecutor onRemoteDatabase() + { + return database; + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + database = closeAfterClass(new TestingTeradataServer(ClearScapeEnvironmentUtils.generateUniqueEnvName(getClass()))); + // Register this specific instance for this test class + return TeradataQueryRunner.builder(database).setInitialTables(REQUIRED_TPCH_TABLES).build(); + } + + @Override + protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) + { + return switch (connectorBehavior) { + case SUPPORTS_ADD_COLUMN, + SUPPORTS_AGGREGATION_PUSHDOWN, + SUPPORTS_COMMENT_ON_COLUMN, + SUPPORTS_COMMENT_ON_TABLE, + SUPPORTS_CREATE_MATERIALIZED_VIEW, + SUPPORTS_CREATE_TABLE_WITH_COLUMN_COMMENT, + SUPPORTS_CREATE_TABLE_WITH_TABLE_COMMENT, + SUPPORTS_CREATE_VIEW, + SUPPORTS_DELETE, + SUPPORTS_DEREFERENCE_PUSHDOWN, + SUPPORTS_DROP_COLUMN, + SUPPORTS_DROP_SCHEMA_CASCADE, + SUPPORTS_INSERT, + SUPPORTS_JOIN_PUSHDOWN, + SUPPORTS_JOIN_PUSHDOWN_WITH_DISTINCT_FROM, + SUPPORTS_JOIN_PUSHDOWN_WITH_VARCHAR_INEQUALITY, + SUPPORTS_LIMIT_PUSHDOWN, + SUPPORTS_MAP_TYPE, + SUPPORTS_MERGE, + SUPPORTS_NATIVE_QUERY, + SUPPORTS_NEGATIVE_DATE, + SUPPORTS_PREDICATE_ARITHMETIC_EXPRESSION_PUSHDOWN, + SUPPORTS_PREDICATE_EXPRESSION_PUSHDOWN, + SUPPORTS_PREDICATE_PUSHDOWN, + SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY, + SUPPORTS_RENAME_COLUMN, + SUPPORTS_RENAME_SCHEMA, + SUPPORTS_RENAME_TABLE, + SUPPORTS_ROW_LEVEL_DELETE, + SUPPORTS_ROW_TYPE, + SUPPORTS_SET_COLUMN_TYPE, + SUPPORTS_TOPN_PUSHDOWN, + SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR, + SUPPORTS_TRUNCATE, + SUPPORTS_UPDATE -> false; + case SUPPORTS_CREATE_SCHEMA, + SUPPORTS_CREATE_TABLE -> true; + default -> super.hasBehavior(connectorBehavior); + }; + } + + @AfterAll + public void cleanupTestDatabase() + { + database = null; + } + + @Override + protected OptionalInt maxSchemaNameLength() + { + return OptionalInt.of(TERADATA_OBJECT_NAME_LIMIT); + } + + @Override // Override because the expected error message is different + protected void verifySchemaNameLengthFailurePermissible(Throwable e) + { + assertThat(e).hasMessage(format("Schema name must be shorter than or equal to '%s' characters but got '%s'", TERADATA_OBJECT_NAME_LIMIT, TERADATA_OBJECT_NAME_LIMIT + 1)); + } + + @Override // Override because Teradata Object name limit is 128 characters + protected OptionalInt maxColumnNameLength() + { + return OptionalInt.of(TERADATA_OBJECT_NAME_LIMIT); + } + + @Override // Override because the expected error message is different + protected void verifyColumnNameLengthFailurePermissible(Throwable e) + { + assertThat(e).hasMessageMatching(format("Column name must be shorter than or equal to '%s' characters but got '%s': '.*'", TERADATA_OBJECT_NAME_LIMIT, + TERADATA_OBJECT_NAME_LIMIT + 1)); + } + + @Override // Override to skip the data mapping smoke test + @Test + public void testDataMappingSmokeTest() + { + skipTestUnless(false); + } + + @Override // Override because Teradata Table name limit is 128 characters + protected OptionalInt maxTableNameLength() + { + return OptionalInt.of(TERADATA_OBJECT_NAME_LIMIT); + } + + @Override // Override because the expected error message is different + protected void verifyTableNameLengthFailurePermissible(Throwable e) + { + assertThat(e).hasMessageMatching(format("Table name must be shorter than or equal to '%s' characters but got '%s'", TERADATA_OBJECT_NAME_LIMIT, + TERADATA_OBJECT_NAME_LIMIT + 1)); + } + + @Override // Overriding this test case as Teradata defines varchar with a length. + @Test + public void testVarcharCastToDateInPredicate() + { + String tableName = "varchar_as_date_pred"; + try (TestTable table = newTrinoTable(tableName, "(a varchar(50))", List.of("'999-09-09'", "'1005-09-09'", "'2005-06-06'", "'2005-06-6'", "'2005-6-06'", "'2005-6-6'", "' " + + "2005-06-06'", "'2005-06-06 '", "' +2005-06-06'", "'02005-06-06'", "'2005-09-06'", "'2005-09-6'", "'2005-9-06'", "'2005-9-6'", "' 2005-09-06'", "'2005-09-06 '", + "' +2005-09-06'", "'02005-09-06'", "'2005-09-09'", "'2005-09-9'", "'2005-9-09'", "'2005-9-9'", "' 2005-09-09'", "'2005-09-09 '", "' +2005-09-09'", "'02005-09-09" + + "'", "'2005-09-10'", "'2005-9-10'", "' 2005-09-10'", "'2005-09-10 '", "' +2005-09-10'", "'02005-09-10'", "'2005-09-20'", "'2005-9-20'", "' 2005-09-20'", + "'2005-09-20 '", "' +2005-09-20'", "'02005-09-20'", "'9999-09-09'", "'99999-09-09'"))) { + for (String date : List.of("2005-09-06", "2005-09-09", "2005-09-10")) { + for (String operator : List.of("=", "<=", "<", ">", ">=", "!=", "IS DISTINCT FROM", "IS NOT DISTINCT FROM")) { + assertThat(query("SELECT a FROM %s WHERE CAST(a AS date) %s DATE '%s'".formatted(table.getName(), operator, date))).hasCorrectResultsRegardlessOfPushdown(); + } + } + } + try (TestTable table = newTrinoTable(tableName, "(a varchar(50))", List.of("'2005-06-bad-date'", "'2005-09-10'"))) { + assertThat(query("SELECT a FROM %s WHERE CAST(a AS date) < DATE '2005-09-10'".formatted(table.getName()))).failure().hasMessage("Value cannot be cast to date: " + + "2005-06-bad-date"); + verifyResultOrFailure(query("SELECT a FROM %s WHERE CAST(a AS date) = DATE '2005-09-10'".formatted(table.getName())), + queryAssert -> queryAssert.skippingTypesCheck().matches("VALUES '2005-09-10'"), failureAssert -> failureAssert.hasMessage("Value cannot be cast to date: " + + "2005-06-bad-date")); + } + try (TestTable table = newTrinoTable(tableName, "(a varchar(50))", List.of("'2005-09-10'"))) { + // 2005-09-01, when written as 2005-09-1, is a prefix of an existing data point: 2005-09-10 + assertThat(query("SELECT a FROM %s WHERE CAST(a AS date) != DATE '2005-09-01'".formatted(table.getName()))).skippingTypesCheck().matches("VALUES '2005-09-10'"); + } + } + + // Tests CREATE TABLE AS SELECT functionality with Teradata syntax + // Overridden to handle Teradata's specific "WITH DATA" syntax for table creation + @Override + @Test + public void testCreateTableAsSelect() + { + String tableName = "test_ctas" + randomNameSuffix(); + assertUpdate("CREATE TABLE IF NOT EXISTS " + tableName + " AS SELECT name, regionkey FROM nation", "SELECT count(*) FROM nation"); + assertTableColumnNames(tableName, "name", "regionkey"); + assertThat(getTableComment(tableName)).isNull(); + assertUpdate("DROP TABLE " + tableName); + + // Some connectors support CREATE TABLE AS but not the ordinary CREATE TABLE. Let's test CTAS IF NOT EXISTS with a table that is guaranteed to exist. + assertUpdate("CREATE TABLE IF NOT EXISTS nation AS SELECT nationkey, regionkey FROM nation", 0); + assertTableColumnNames("nation", "nationkey", "name", "regionkey", "comment"); + + assertCreateTableAsSelect("SELECT nationkey, name, regionkey FROM nation", "SELECT count(*) FROM nation"); + + assertCreateTableAsSelect("SELECT mktsegment, sum(acctbal) x FROM customer GROUP BY mktsegment", "SELECT count(DISTINCT mktsegment) FROM customer"); + + assertCreateTableAsSelect("SELECT count(*) x FROM nation JOIN region ON nation.regionkey = region.regionkey", "SELECT 1"); + + assertCreateTableAsSelect("SELECT nationkey FROM nation ORDER BY nationkey LIMIT 10", "SELECT 10"); + + // Tests for CREATE TABLE with UNION ALL: exercises PushTableWriteThroughUnion optimizer + + assertCreateTableAsSelect("SELECT name, nationkey, regionkey FROM nation WHERE nationkey % 2 = 0 UNION ALL " + "SELECT name, nationkey, regionkey FROM nation WHERE " + + "nationkey % 2 = 1", "SELECT name, nationkey, regionkey FROM nation", "SELECT count(*) FROM nation"); + + assertCreateTableAsSelect(Session.builder(getSession()).setSystemProperty("redistribute_writes", "true").build(), "SELECT CAST(nationkey AS BIGINT) nationkey, regionkey " + + "FROM nation UNION ALL " + "SELECT 1234567890, 123", "SELECT nationkey, regionkey FROM nation UNION ALL " + "SELECT 1234567890, 123", "SELECT count(*) + 1 FROM " + + "nation"); + + assertCreateTableAsSelect(Session.builder(getSession()).setSystemProperty("redistribute_writes", "false").build(), "SELECT CAST(nationkey AS BIGINT) nationkey, regionkey" + + " FROM nation UNION ALL " + "SELECT 1234567890, 123", "SELECT nationkey, regionkey FROM nation UNION ALL " + "SELECT 1234567890, 123", "SELECT count(*) + 1 FROM " + + "nation"); + + tableName = "test_ctas" + randomNameSuffix(); + assertThat(query("EXPLAIN ANALYZE CREATE TABLE " + tableName + " AS SELECT name FROM nation")).succeeds(); + assertThat(query("SELECT * from " + tableName)).matches("SELECT name FROM nation"); + assertUpdate("DROP TABLE " + tableName); + } + + @Override // Overriding this test case as Teradata does not support negative dates. + @Test + public void testDateYearOfEraPredicate() + { + assertQuery("SELECT orderdate FROM orders WHERE orderdate = DATE '1997-09-14'", "VALUES DATE '1997-09-14'"); + } + + @Override // Override this test case as Teradata has different syntax for creating tables with AS SELECT statement. + @Test + public void verifySupportsRowLevelUpdateDeclaration() + { + String testTableName = "test_supports_update"; + try (TestTable table = newTrinoTable(testTableName, "AS ( SELECT * FROM nation) WITH DATA")) { + assertQueryFails("UPDATE " + table.getName() + " SET nationkey = nationkey * 100 WHERE regionkey = 2", "This connector does not support modifying table rows"); + } + } + + @Override // Overriding this test case as Teradata doesn't have support to (k, v) AS VALUES in insert statement + @Test + public void testCharVarcharComparison() + { + String testTableName = "test_char_varchar"; + try (TestTable table = newTrinoTable(testTableName, "(k int, v char(3))", List.of("-1, CAST(NULL AS char(3))", "3, CAST(' ' AS char(3))", "6, CAST('x ' AS char(3))"))) { + assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(2))", "VALUES (3, ' ')"); + assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS varchar(4))", "VALUES (3, ' ')"); + assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST('x ' AS varchar(2))", "VALUES (6, 'x ')"); + } + } + + @Override // Overriding this test case as Teradata doesn't have support to (k, v) AS VALUES in insert statement + @Test + public void testVarcharCharComparison() + { + try (TestTable table = newTrinoTable("test_varchar_char", "(k int, v char(3))", List.of("-1, CAST(NULL AS varchar(3))", "0, CAST('' AS varchar(3))", "1, CAST(' ' AS" + + " varchar(3))", "2, CAST(' ' AS varchar(3))", "3, CAST(' ' AS varchar(3))", "4, CAST('x' AS varchar(3))", "5, CAST('x ' AS varchar(3))", + "6, CAST('x ' AS " + "varchar(3))"))) { + // Teradata's CHAR type automatically pads values with spaces to the defined length + assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST(' ' AS char(2))", "VALUES (0, ' '), (1, ' '), (2, ' '), (3, ' ')"); + assertQuery("SELECT k, v FROM " + table.getName() + " WHERE v = CAST('x ' AS char(2))", "VALUES (4, 'x '), (5, 'x '), (6, 'x ')"); + } + } + + // Filters data mapping test data for Teradata compatibility + // Overridden to exclude data types that Teradata doesn't support or handles differently + @Override + protected Optional filterDataMappingSmokeTestData(DataMappingTestSetup dataMappingTestSetup) + { + String typeName = dataMappingTestSetup.getTrinoTypeName(); + return switch (typeName) { + // skipping date as during julian->gregorian date is handled differently in Teradata. tinyint, double and varchar with unbounded (need to handle special characters) + // is skipped and will handle it while improving + // write functionalities. + case "boolean", "tinyint", "date", "real", "double", "varchar", "time", "time(6)", "timestamp", "timestamp(6)", "varbinary", "timestamp(3) with time zone", + "timestamp(6) with time zone", "U&'a \\000a newline'" -> Optional.empty(); + default -> Optional.of(dataMappingTestSetup); + }; + } + + @Override + @Test + public void testTimestampWithTimeZoneCastToDatePredicate() + { + abort("Skipping as connector does not support Timestamp with Time Zone data type"); + } + + @Override + @Test + public void testTimestampWithTimeZoneCastToTimestampPredicate() + { + abort("Skipping as connector does not support Timestamp with Time Zone data type"); + } + + @Override + @Test + public void testRenameSchema() + { + abort("Skipping as connector does not support RENAME SCHEMA"); + } + + @Override + @Test + public void testColumnName() + { + abort("Skipping as connector does not support column level write operations"); + } + + @Override + @Test + public void testCreateTableAsSelectWithUnicode() + { + abort("Skipping as connector does not support creating table with UNICODE characters"); + } + + @Override + @Test + public void testUpdateNotNullColumn() + { + abort("Skipping as connector does not support insert operations"); + } + + @Override + @Test + public void testWriteBatchSizeSessionProperty() + { + abort("Skipping as connector does not support insert operations"); + } + + @Override + @Test + public void testInsertWithoutTemporaryTable() + { + abort("Skipping as connector does not support insert operations"); + } + + @Override + @Test + public void testWriteTaskParallelismSessionProperty() + { + abort("Skipping as connector does not support insert operations"); + } + + @Override + @Test + public void testInsertIntoNotNullColumn() + { + abort("Skipping as connector does not support insert operations"); + } + + @Override + @Test + public void testDropSchemaCascade() + { + abort("Skipping as connector does not support dropping schemas with CASCADE option"); + } + + @Override + @Test + public void testAddColumn() + { + abort("Skipping as connector does not support column level write operations"); + } + + @Override + @Test + public void testDropNonEmptySchemaWithTable() + { + abort("Skipping as connector does not support drop schemas"); + } + + @Override + @Test + public void verifySupportsUpdateDeclaration() + { + abort("Skipping as connector does not support update operations"); + } + + @Override + @Test + public void testDropNotNullConstraint() + { + abort("Skipping as connector does not support dropping a not null constraint"); + } + + @Override + @Test + public void testExecuteProcedureWithInvalidQuery() + { + abort("Skipping as connector does not support execute procedure"); + } + + @Override + @Test + public void testCreateTableAsSelectNegativeDate() + { + abort("Skipping as connector does not support creating table with negative date"); + } + + // Creates CTAS queries with proper session and row count validation + // Overridden to use Teradata's "WITH DATA" syntax for CREATE TABLE AS SELECT statements + @Override + protected void assertCreateTableAsSelect(Session session, String query, String expectedQuery, String rowCountQuery) + { + String table = "test_ctas_" + TestingNames.randomNameSuffix(); + assertUpdate(session, "CREATE TABLE " + table + " AS ( " + query + ") WITH DATA", rowCountQuery); + assertQuery(session, "SELECT * FROM " + table, expectedQuery); + assertUpdate(session, "DROP TABLE " + table); + assertThat(getQueryRunner().tableExists(session, table)).isFalse(); + } + + // Creates new Trino test tables with proper schema handling + // Overridden to handle Teradata's schema.table naming format and table creation syntax + @Override + protected TestTable newTrinoTable(String namePrefix, @Language("SQL") String tableDefinition, List rowsToInsert) + { + String tableName; + + // Check if namePrefix already contains schema (contains a dot) + if (namePrefix.contains(".")) { + // namePrefix already has schema.tablename format + tableName = namePrefix; + } + else { + // Append current schema to namePrefix + String schemaName = getSession().getSchema().orElseThrow(); + tableName = schemaName + "." + namePrefix; + } + return new TestTable(database, tableName, tableDefinition, rowsToInsert); + } + + @Test + public void testTeradataNumberDataType() + { + try (TestTable table = newTrinoTable("test_number", "(id INTEGER, " + "number_col NUMBER(10,2), " + "number_default NUMBER, " + "number_large NUMBER(38,10))", List.of( + "1, CAST(12345.67 AS NUMBER(10,2)), CAST(999999999999999 AS NUMBER), CAST(1234567890123456789012345678.1234567890 AS NUMBER(38,10))", "2, CAST(-99999.99 AS " + + "NUMBER(10,2)), CAST(-123456789012345 AS NUMBER), CAST(-9999999999999999999999999999.9999999999 AS NUMBER(38,10))", + "3, CAST(0.00 AS NUMBER(10,2)), CAST" + "(0 AS NUMBER), CAST(0.0000000000 AS NUMBER(38,10))"))) { + assertThat(query(format("SELECT number_col FROM %s WHERE id = 1", table.getName()))).matches("VALUES CAST(12345.67 AS DECIMAL(10,2))"); + assertThat(query(format("SELECT number_default FROM %s WHERE id = 1", table.getName()))).matches("VALUES CAST(999999999999999 AS DECIMAL(38,0))"); + assertThat(query(format("SELECT number_large FROM %s WHERE id = 1", table.getName()))).matches("VALUES CAST(1234567890123456789012345678.1234567890 AS DECIMAL(38,10)" + + ")"); + assertThat(query(format("SELECT number_col FROM %s WHERE id = 2", table.getName()))).matches("VALUES CAST(-99999.99 AS DECIMAL(10,2))"); + assertThat(query(format("SELECT number_col FROM %s WHERE id = 3", table.getName()))).matches("VALUES CAST(0.00 AS DECIMAL(10,2))"); + } + } + + @Test + public void testTeradataCharacterDataType() + { + try (TestTable table = newTrinoTable("test_character", "(id INTEGER, " + "char_col CHARACTER(5), " + "char_default CHARACTER, " + "char_large CHARACTER(100))", List.of( + "1, CAST('HELLO' AS CHARACTER(5)), CAST('A' AS CHARACTER), CAST('TERADATA' AS CHARACTER(100))", + "2, CAST('WORLD' AS CHARACTER(5)), CAST('B' AS CHARACTER), CAST" + "('CHARACTER' AS CHARACTER(100))", "3, CAST('' AS CHARACTER(5)), CAST('C' AS CHARACTER), CAST" + + "('' AS CHARACTER(100))"))) { + assertThat(query(format("SELECT char_col FROM %s WHERE id = 1", table.getName()))).matches("VALUES CAST('HELLO' AS CHAR(5))"); + assertThat(query(format("SELECT char_default FROM %s WHERE id = 1", table.getName()))).matches("VALUES CAST('A' AS CHAR(1))"); + assertThat(query(format("SELECT char_large FROM %s WHERE id = 1", table.getName()))).matches("VALUES CAST('TERADATA' AS CHAR(100))"); + assertThat(query(format("SELECT char_col FROM %s WHERE id = 3", table.getName()))).matches("VALUES CAST('' AS CHAR(5))"); + } + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TeradataQueryRunner.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TeradataQueryRunner.java new file mode 100644 index 000000000000..b38eb153dfe5 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TeradataQueryRunner.java @@ -0,0 +1,126 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration; + +import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import io.airlift.log.Level; +import io.airlift.log.Logger; +import io.airlift.log.Logging; +import io.trino.Session; +import io.trino.metadata.QualifiedObjectName; +import io.trino.plugin.integration.clearscape.ClearScapeEnvironmentUtils; +import io.trino.plugin.teradata.TeradataPlugin; +import io.trino.plugin.tpch.TpchPlugin; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.QueryRunner; +import io.trino.tpch.TpchTable; +import org.assertj.core.api.ObjectAssert; +import org.intellij.lang.annotations.Language; + +import java.util.List; +import java.util.Locale; + +import static io.trino.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; + +public final class TeradataQueryRunner +{ + private TeradataQueryRunner() {} + + public static Builder builder(TestingTeradataServer server) + { + return new Builder(server); + } + + public static void main(String[] args) + throws Exception + { + Logging logger = Logging.initialize(); + logger.setLevel("io.trino.plugin.teradata", Level.DEBUG); + logger.setLevel("io.trino", Level.INFO); + TestingTeradataServer server = new TestingTeradataServer(ClearScapeEnvironmentUtils.generateUniqueEnvName(TeradataQueryRunner.class)); + QueryRunner queryRunner = builder(server).addCoordinatorProperty("http-server.http.port", "8080").setInitialTables(TpchTable.getTables()).build(); + + Logger log = Logger.get(TeradataQueryRunner.class); + log.info("======== SERVER STARTED ========"); + log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl()); + } + + public static class Builder + extends DistributedQueryRunner.Builder + { + private final TestingTeradataServer server; + private List> initialTables = ImmutableList.of(); + + protected Builder(TestingTeradataServer server) + { + super(testSessionBuilder().setCatalog("teradata").setSchema(server.getDatabaseName()).build()); + this.server = requireNonNull(server, "server is null"); + } + + public void copyTable(QueryRunner queryRunner, QualifiedObjectName table, Session session) + { + @Language("SQL") String sql = String.format("CREATE TABLE %s AS SELECT * FROM %s", table.objectName(), table); + queryRunner.execute(session, sql); + ((ObjectAssert) assertThat(queryRunner.execute(session, "SELECT count(*) FROM " + table.objectName()).getOnlyValue()).as("Table is not loaded properly: %s", new Object[] { + table.objectName()})).isEqualTo(queryRunner.execute(session, "SELECT count(*) FROM " + table).getOnlyValue()); + } + + public void copyTpchTables(QueryRunner queryRunner, String sourceCatalog, String sourceSchema, Session session, Iterable> tables) + { + for (TpchTable table : tables) { + copyTable(queryRunner, sourceCatalog, sourceSchema, table.getTableName().toLowerCase(Locale.ENGLISH), session); + } + } + + public void copyTpchTables(QueryRunner queryRunner, String sourceCatalog, String sourceSchema, Iterable> tables) + { + copyTpchTables(queryRunner, sourceCatalog, sourceSchema, queryRunner.getDefaultSession(), tables); + } + + public void copyTable(QueryRunner queryRunner, String sourceCatalog, String sourceSchema, String sourceTable, Session session) + { + QualifiedObjectName table = new QualifiedObjectName(sourceCatalog, sourceSchema, sourceTable); + if (!server.isTableExists(sourceTable)) { + copyTable(queryRunner, table, session); + } + } + + @CanIgnoreReturnValue + public Builder setInitialTables(Iterable> initialTables) + { + this.initialTables = ImmutableList.copyOf(requireNonNull(initialTables, "initialTables is null")); + return this; + } + + @Override + public DistributedQueryRunner build() + throws Exception + { + super.setAdditionalSetup(runner -> { + runner.installPlugin(new TpchPlugin()); + runner.createCatalog("tpch", "tpch"); + + runner.installPlugin(new TeradataPlugin()); + runner.createCatalog("teradata", "teradata", server.getCatalogProperties()); + + copyTpchTables(runner, "tpch", TINY_SCHEMA_NAME, initialTables); + }); + return super.build(); + } + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TestTeradataTypeMapping.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TestTeradataTypeMapping.java new file mode 100644 index 000000000000..c31af79efddb --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TestTeradataTypeMapping.java @@ -0,0 +1,287 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration; + +import io.trino.plugin.integration.clearscape.ClearScapeEnvironmentUtils; +import io.trino.testing.AbstractTestQueryFramework; +import io.trino.testing.QueryRunner; +import io.trino.testing.datatype.CreateAndInsertDataSetup; +import io.trino.testing.datatype.DataSetup; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +import java.sql.SQLException; + +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.CharType.createCharType; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.DoubleType.DOUBLE; +import static io.trino.spi.type.IntegerType.INTEGER; +import static io.trino.spi.type.SmallintType.SMALLINT; +import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.VarcharType.createVarcharType; +import static io.trino.testing.datatype.SqlDataTypeTest.create; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; + +final class TestTeradataTypeMapping + extends AbstractTestQueryFramework +{ + private final String envName; + private TestingTeradataServer database; + + public TestTeradataTypeMapping() + { + envName = ClearScapeEnvironmentUtils.generateUniqueEnvName(TestTeradataTypeMapping.class); + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + database = closeAfterClass(new TestingTeradataServer(envName)); + // Register this specific instance for this test class + return TeradataQueryRunner.builder(database).build(); + } + + @AfterAll + void cleanupTestClass() + { + database = null; + } + + @Test + void testByteint() + { + create() + .addRoundTrip("byteint", "0", TINYINT, "CAST(0 AS TINYINT)") + .addRoundTrip("byteint", "127", TINYINT, "CAST(127 AS TINYINT)") + .addRoundTrip("byteint", "-128", TINYINT, "CAST(-128 AS TINYINT)") + .addRoundTrip("byteint", "null", TINYINT, "CAST(null AS TINYINT)") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("byteint")); + } + + @Test + void testSmallint() + { + create() + .addRoundTrip("smallint", "0", SMALLINT, "CAST(0 AS SMALLINT)") + .addRoundTrip("smallint", "32767", SMALLINT, "CAST(32767 AS SMALLINT)") + .addRoundTrip("smallint", "-32768", SMALLINT, "CAST(-32768 AS SMALLINT)") + .addRoundTrip("smallint", "null", SMALLINT, "CAST(null AS SMALLINT)") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("smallint")); + } + + @Test + void testInteger() + { + create() + .addRoundTrip("integer", "0", INTEGER, "0") + .addRoundTrip("integer", "2147483647", INTEGER, "2147483647") + .addRoundTrip("integer", "-2147483648", INTEGER, "-2147483648") + .addRoundTrip("integer", "NULL", INTEGER, "CAST(NULL AS INTEGER)") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("integer")); + } + + @Test + void testBigint() + { + create() + .addRoundTrip("bigint", "0", BIGINT, "CAST(0 AS BIGINT)") + .addRoundTrip("bigint", "9223372036854775807", BIGINT, "9223372036854775807") + .addRoundTrip("bigint", "-9223372036854775808", BIGINT, "-9223372036854775808") + .addRoundTrip("bigint", "NULL", BIGINT, "CAST(NULL AS BIGINT)") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("bigint")); + } + + @Test + void testFloat() + { + create() + .addRoundTrip("float", "0", DOUBLE, "CAST(0 AS DOUBLE)") + .addRoundTrip("real", "0", DOUBLE, "CAST(0 AS DOUBLE)") + .addRoundTrip("double precision", "0", DOUBLE, "CAST(0 AS DOUBLE)") + .addRoundTrip("float", "1.797e308", DOUBLE, "1.797e308") + .addRoundTrip("real", "1.797e308", DOUBLE, "1.797e308") + .addRoundTrip("double precision", "1.797e308", DOUBLE, "1.797e308") + .addRoundTrip("float", "2.226e-308", DOUBLE, "2.226e-308") + .addRoundTrip("real", "2.226e-308", DOUBLE, "2.226e-308") + .addRoundTrip("double precision", "2.226e-308", DOUBLE, "2.226e-308") + .addRoundTrip("float", "NULL", DOUBLE, "CAST(NULL AS DOUBLE)") + .addRoundTrip("real", "NULL", DOUBLE, "CAST(NULL AS DOUBLE)") + .addRoundTrip("double precision", "NULL", DOUBLE, "CAST(NULL AS DOUBLE)") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("float")); + } + + @Test + void testDecimal() + { + create() + .addRoundTrip("decimal(3, 0)", "0", createDecimalType(3, 0), "CAST('0' AS decimal(3, 0))") + .addRoundTrip("numeric(3, 0)", "0", createDecimalType(3, 0), "CAST('0' AS decimal(3, 0))") + .addRoundTrip("decimal(3, 1)", "0.0", createDecimalType(3, 1), "CAST('0.0' AS decimal(3, 1))") + .addRoundTrip("numeric(3, 1)", "0.0", createDecimalType(3, 1), "CAST('0.0' AS decimal(3, 1))") + .addRoundTrip("decimal(1, 0)", "1", createDecimalType(1, 0), "CAST('1' AS decimal(1, 0))") + .addRoundTrip("numeric(1, 0)", "1", createDecimalType(1, 0), "CAST('1' AS decimal(1, 0))") + .addRoundTrip("decimal(1, 0)", "-1", createDecimalType(1, 0), "CAST('-1' AS decimal(1, 0))") + .addRoundTrip("numeric(1, 0)", "-1", createDecimalType(1, 0), "CAST('-1' AS decimal(1, 0))") + .addRoundTrip("decimal(3, 0)", "1", createDecimalType(3, 0), "CAST('1' AS decimal(3, 0))") + .addRoundTrip("numeric(3, 0)", "1", createDecimalType(3, 0), "CAST('1' AS decimal(3, 0))") + .addRoundTrip("decimal(3, 0)", "-1", createDecimalType(3, 0), "CAST('-1' AS decimal(3, 0))") + .addRoundTrip("numeric(3, 0)", "-1", createDecimalType(3, 0), "CAST('-1' AS decimal(3, 0))") + .addRoundTrip("decimal(3, 0)", "123", createDecimalType(3, 0), "CAST('123' AS decimal(3, 0))") + .addRoundTrip("numeric(3, 0)", "123", createDecimalType(3, 0), "CAST('123' AS decimal(3, 0))") + .addRoundTrip("decimal(3, 0)", "-123", createDecimalType(3, 0), "CAST('-123' AS decimal(3, 0))") + .addRoundTrip("numeric(3, 0)", "-123", createDecimalType(3, 0), "CAST('-123' AS decimal(3, 0))") + .addRoundTrip("decimal(3, 1)", "10.0", createDecimalType(3, 1), "CAST('10.0' AS decimal(3, 1))") + .addRoundTrip("numeric(3, 1)", "10.0", createDecimalType(3, 1), "CAST('10.0' AS decimal(3, 1))") + .addRoundTrip("decimal(3, 1)", "12.3", createDecimalType(3, 1), "CAST('12.3' AS decimal(3, 1))") + .addRoundTrip("numeric(3, 1)", "12.3", createDecimalType(3, 1), "CAST('12.3' AS decimal(3, 1))") + .addRoundTrip("decimal(3, 1)", "-12.3", createDecimalType(3, 1), "CAST('-12.3' AS decimal(3, 1))") + .addRoundTrip("numeric(3, 1)", "-12.3", createDecimalType(3, 1), "CAST('-12.3' AS decimal(3, 1))") + .addRoundTrip("decimal(38, 0)", "12345678901234567890123456789012345678", createDecimalType(38, 0), "CAST('12345678901234567890123456789012345678' AS decimal(38, 0))") + .addRoundTrip("numeric(38, 0)", "12345678901234567890123456789012345678", createDecimalType(38, 0), "CAST('12345678901234567890123456789012345678' AS decimal(38, 0))") + .addRoundTrip("decimal(38, 0)", "-12345678901234567890123456789012345678", createDecimalType(38, 0), "CAST('-12345678901234567890123456789012345678' AS decimal(38, 0))") + .addRoundTrip("numeric(38, 0)", "-12345678901234567890123456789012345678", createDecimalType(38, 0), "CAST('-12345678901234567890123456789012345678' AS decimal(38, 0))") + .addRoundTrip("decimal(1, 0)", "null", createDecimalType(1, 0), "CAST(null AS decimal(1, 0))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("decimal")); + } + + @Test + void testNumber() + { + create() + .addRoundTrip("numeric(3)", "0", createDecimalType(3, 0), "CAST('0' AS decimal(3, 0))") + .addRoundTrip("number(5,2)", "0", createDecimalType(5, 2), "CAST('0' AS decimal(5, 2))") + .addRoundTrip("number(38)", "0", createDecimalType(38, 0), "CAST('0' AS decimal(38, 0))") + .addRoundTrip("number(38,2)", "123456789012345678901234567890123456.78", createDecimalType(38, 2), "CAST('123456789012345678901234567890123456.78' AS decimal(38, 2))") + .addRoundTrip("numeric(38)", "12345678901234567890123456789012345678", createDecimalType(38, 0), "CAST('12345678901234567890123456789012345678' AS decimal(38, 0))") + .addRoundTrip("numeric(3)", "null", createDecimalType(3, 0), "CAST(null AS decimal(3, 0))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("number")); + } + + @Test + void testChar() + { + create() + .addRoundTrip("char(3)", "''", createCharType(3), "CAST('' AS char(3))") + .addRoundTrip("char(3)", "' '", createCharType(3), "CAST(' ' AS char(3))") + .addRoundTrip("char(3)", "' '", createCharType(3), "CAST(' ' AS char(3))") + .addRoundTrip("char(3)", "' '", createCharType(3), "CAST(' ' AS char(3))") + .addRoundTrip("char(3)", "'A'", createCharType(3), "CAST('A' AS char(3))") + .addRoundTrip("char(3)", "'A '", createCharType(3), "CAST('A ' AS char(3))") + .addRoundTrip("char(3)", "' B '", createCharType(3), "CAST(' B ' AS char(3))") + .addRoundTrip("char(3)", "' C'", createCharType(3), "CAST(' C' AS char(3))") + .addRoundTrip("char(3)", "'AB'", createCharType(3), "CAST('AB' AS char(3))") + .addRoundTrip("char(3)", "'ABC'", createCharType(3), "CAST('ABC' AS char(3))") + .addRoundTrip("char(3)", "'A C'", createCharType(3), "CAST('A C' AS char(3))") + .addRoundTrip("char(3)", "' BC'", createCharType(3), "CAST(' BC' AS char(3))") + .addRoundTrip("char(3)", "null", createCharType(3), "CAST(null AS char(3))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("char")); + String tmode = database.getTMode(); + if (tmode.equals("TERA")) { + // truncation + create() + .addRoundTrip("char(3)", "'ABCD'", createCharType(3), "CAST('ABCD' AS char(3))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("chart")); + } + else { + // Error on truncation + assertThatThrownBy(() -> + create() + .addRoundTrip("char(3)", "'ABCD'", createCharType(3), "CAST('ABCD' AS char(3))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("chart"))) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(SQLException.class) + .cause() + .hasMessageContaining("Right truncation of string data"); + } + // max-size + create() + .addRoundTrip("char(64000)", "'max'", createCharType(64000), "CAST('max' AS char(64000))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("charl")); + } + + @Test + void testVarchar() + { + create() + .addRoundTrip("varchar(32)", "''", createVarcharType(32), "CAST('' AS varchar(32))") + .addRoundTrip("varchar(32)", "' '", createVarcharType(32), "CAST(' ' AS varchar(32))") + .addRoundTrip("varchar(32)", "' '", createVarcharType(32), "CAST(' ' AS varchar(32))") + .addRoundTrip("varchar(32)", "' '", createVarcharType(32), "CAST(' ' AS varchar(32))") + .addRoundTrip("varchar(32)", "' '", createVarcharType(32), "CAST(' ' AS varchar(32))") + .addRoundTrip("varchar(32)", "'A'", createVarcharType(32), "CAST('A' AS varchar(32))") + .addRoundTrip("varchar(32)", "'A '", createVarcharType(32), "CAST('A ' AS varchar(32))") + .addRoundTrip("varchar(32)", "' B '", createVarcharType(32), "CAST(' B ' AS varchar(32))") + .addRoundTrip("varchar(32)", "' C'", createVarcharType(32), "CAST(' C' AS varchar(32))") + .addRoundTrip("varchar(32)", "'AB'", createVarcharType(32), "CAST('AB' AS varchar(32))") + .addRoundTrip("varchar(32)", "'ABC'", createVarcharType(32), "CAST('ABC' AS varchar(32))") + .addRoundTrip("varchar(32)", "'A C'", createVarcharType(32), "CAST('A C' AS varchar(32))") + .addRoundTrip("varchar(32)", "' BC'", createVarcharType(32), "CAST(' BC' AS varchar(32))") + .addRoundTrip("varchar(32)", "null", createVarcharType(32), "CAST(null AS varchar(32))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("varchar")); + String teraMode = database.getTMode(); + if (teraMode.equals("TERA")) { + // truncation + create() + .addRoundTrip("varchar(3)", "'ABCD'", createVarcharType(3), "CAST('ABCD' AS varchar(3))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("varchart")); + } + else { + // Error on truncation + assertThatThrownBy(() -> + create() + .addRoundTrip("varchar(3)", "'ABCD'", createVarcharType(3), "CAST('ABCD' AS varchar(3))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("varchart"))) + .isInstanceOf(RuntimeException.class) + .hasCauseInstanceOf(SQLException.class) + .cause() + .hasMessageContaining("Right truncation of string data"); + } + // max-size + create() + .addRoundTrip("long varchar", "'max'", createVarcharType(64000), "CAST('max' AS varchar(64000))") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("varcharl")); + } + + @Test + void testDate() + { + create() + .addRoundTrip("date", "DATE '0001-01-01'", DATE, "DATE '0001-01-01'") + .addRoundTrip("date", "DATE '0012-12-12'", DATE, "DATE '0012-12-12'") + .addRoundTrip("date", "DATE '1500-01-01'", DATE, "DATE '1500-01-01'") + .addRoundTrip("date", "DATE '1582-10-04'", DATE, "DATE '1582-10-04'") + .addRoundTrip("date", "DATE '1582-10-15'", DATE, "DATE '1582-10-15'") + .addRoundTrip("date", "DATE '1952-04-03'", DATE, "DATE '1952-04-03'") + .addRoundTrip("date", "DATE '1970-01-01'", DATE, "DATE '1970-01-01'") + .addRoundTrip("date", "DATE '1970-02-03'", DATE, "DATE '1970-02-03'") + .addRoundTrip("date", "DATE '1970-01-01'", DATE, "DATE '1970-01-01'") + .addRoundTrip("date", "DATE '1983-04-01'", DATE, "DATE '1983-04-01'") + .addRoundTrip("date", "DATE '1983-10-01'", DATE, "DATE '1983-10-01'") + .addRoundTrip("date", "DATE '2017-07-01'", DATE, "DATE '2017-07-01'") + .addRoundTrip("date", "DATE '2017-01-01'", DATE, "DATE '2017-01-01'") + .addRoundTrip("date", "DATE '2024-02-29'", DATE, "DATE '2024-02-29'") + .addRoundTrip("date", "DATE '9999-12-30'", DATE, "DATE '9999-12-30'") + .addRoundTrip("date", "NULL", DATE, "CAST(NULL AS DATE)") + .execute(getQueryRunner(), teradataJDBCCreateAndInsert("date")); + } + + private DataSetup teradataJDBCCreateAndInsert(String tableNamePrefix) + { + String prefix = String.format("%s.%s", database.getDatabaseName(), tableNamePrefix); + return new CreateAndInsertDataSetup(database, prefix); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TestingTeradataServer.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TestingTeradataServer.java new file mode 100644 index 000000000000..b47c9eb9058f --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/TestingTeradataServer.java @@ -0,0 +1,277 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration; + +import io.trino.plugin.integration.clearscape.ClearScapeSetup; +import io.trino.plugin.integration.clearscape.Model; +import io.trino.plugin.integration.clearscape.Region; +import io.trino.plugin.integration.util.TeradataTestConstants; +import io.trino.plugin.teradata.LogonMechanism; +import io.trino.testing.sql.SqlExecutor; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static io.trino.testing.SystemEnvironmentUtils.isEnvSet; +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; +import static java.util.Objects.requireNonNull; + +public class TestingTeradataServer + implements AutoCloseable, SqlExecutor +{ + private static final int MAX_RETRIES = 3; + private static final long RETRY_DELAY_MS = 1000; + private final Connection connection; + private DatabaseConfig config; + private ClearScapeSetup clearScapeSetup; + + public TestingTeradataServer(String envName) + { + config = DatabaseConfigFactory.create(envName); + String hostName = config.getHostName(); + // Initialize ClearScape Instance and Get the host name from ClearScape API in case config is using clearscape + if (config.isUseClearScape()) { + boolean destroyEnv = false; + if (isEnvSet("CLEARSCAPE_DESTROY_ENV")) { + String destroyEnvValue = requireEnv("CLEARSCAPE_DESTROY_ENV"); + destroyEnv = Boolean.parseBoolean(destroyEnvValue); + } + String region = TeradataTestConstants.ENV_CLEARSCAPE_REGION; + if (isEnvSet("CLEARSCAPE_REGION")) { + String envRegion = requireEnv("CLEARSCAPE_REGION"); + if (Region.isValid(envRegion)) { + region = envRegion; + } + } + clearScapeSetup = new ClearScapeSetup( + requireEnv("CLEARSCAPE_TOKEN"), + requireEnv("CLEARSCAPE_PASSWORD"), + config.getClearScapeEnvName(), + destroyEnv, + region); + Model model = clearScapeSetup.initialize(); + hostName = model.getHostName(); + } + String jdbcUrl = buildJdbcUrl(hostName); + config = config.toBuilder() + .hostName(hostName) + .jdbcUrl(jdbcUrl) + .build(); + connection = createConnection(); + createTestDatabaseIfAbsent(); + } + + private String buildJdbcUrl(String hostName) + { + String baseUrl = String.format("jdbc:teradata://%s/", hostName); + String propertiesString = buildPropertiesString(); + if (propertiesString.isEmpty()) { + return baseUrl; + } + return baseUrl + propertiesString; + } + + private String buildPropertiesString() + { + Map properties = config.getJdbcProperties(); + if (properties == null || properties.isEmpty()) { + return ""; + } + + return properties.entrySet() + .stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(java.util.stream.Collectors.joining(",")); + } + + private Connection createConnection() + { + try { + Class.forName("com.teradata.jdbc.TeraDriver"); + Properties props = buildConnectionProperties(); + return DriverManager.getConnection(config.getJdbcUrl(), props); + } + catch (SQLException | ClassNotFoundException e) { + throw new RuntimeException("Failed to create database connection", e); + } + } + + private Properties buildConnectionProperties() + { + Properties props = new Properties(); + props.put("logmech", config.getLogMech().getMechanism()); + + if (requireNonNull(config.getLogMech()) == LogonMechanism.TD2) { + AuthenticationConfig auth = config.getAuthConfig(); + props.put("username", auth.userName()); + props.put("password", auth.password()); + } + else { + throw new IllegalArgumentException("Unsupported logon mechanism: " + config.getLogMech()); + } + + return props; + } + + public Map getCatalogProperties() + { + Map properties = new HashMap<>(); + properties.put("connection-url", config.getJdbcUrl()); + properties.put("logon-mechanism", config.getLogMech().getMechanism()); + + if (requireNonNull(config.getLogMech()) == LogonMechanism.TD2) { + AuthenticationConfig auth = config.getAuthConfig(); + properties.put("connection-user", auth.userName()); + properties.put("connection-password", auth.password()); + } + + return properties; + } + + public void createTestDatabaseIfAbsent() + { + executeWithRetry(() -> { + if (!schemaExists(config.getDatabaseName())) { + execute(String.format("CREATE DATABASE \"%s\" AS PERM=100e6;", config.getDatabaseName())); + } + }); + } + + public void dropTestDatabaseIfExists() + { + executeWithRetry(() -> { + if (schemaExists(config.getDatabaseName())) { + execute(String.format("DELETE DATABASE \"%s\"", config.getDatabaseName())); + execute(String.format("DROP DATABASE \"%s\"", config.getDatabaseName())); + } + }); + } + + private void executeWithRetry(Runnable operation) + { + int attempts = 0; + while (attempts < MAX_RETRIES) { + try { + operation.run(); + return; // Success, exit retry loop + } + catch (RuntimeException e) { + if (isTeradataError3598(e) && attempts < MAX_RETRIES - 1) { + attempts++; + try { + Thread.sleep(RETRY_DELAY_MS * attempts); // Exponential backoff + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during retry", ie); + } + } + else { + throw e; // Re-throw if not error 3598 or max retries reached + } + } + } + } + + private boolean isTeradataError3598(Exception e) + { + if (e.getCause() instanceof SQLException sqlException) { + return sqlException.getErrorCode() == 3598; + } + return false; + } + + private boolean schemaExists(String schemaName) + { + String query = "SELECT COUNT(1) FROM DBC.DatabasesV WHERE DatabaseName = ?"; + try (PreparedStatement stmt = connection.prepareStatement(query)) { + stmt.setString(1, schemaName); + try (ResultSet rs = stmt.executeQuery()) { + return rs.next() && rs.getInt(1) > 0; + } + } + catch (SQLException e) { + throw new RuntimeException("Failed to check schema existence", e); + } + } + + public boolean isTableExists(String tableName) + { + String query = "SELECT count(1) FROM DBC.TablesV WHERE DataBaseName = ? AND TableName = ?"; + try (PreparedStatement stmt = connection.prepareStatement(query)) { + stmt.setString(1, config.getDatabaseName()); + stmt.setString(2, tableName); + try (ResultSet rs = stmt.executeQuery()) { + return rs.next() && rs.getInt(1) > 0; + } + } + catch (SQLException e) { + throw new RuntimeException("Failed to check table existence: " + e.getMessage(), e); + } + } + + @Override + public void execute(String sql) + { + try (Statement stmt = connection.createStatement()) { + if (config.getDatabaseName() != null && schemaExists(config.getDatabaseName())) { + stmt.execute(String.format("DATABASE \"%s\"", config.getDatabaseName())); + } + stmt.execute(sql); + } + catch (SQLException e) { + throw new RuntimeException("SQL execution failed: " + sql, e); + } + } + + public String getDatabaseName() + { + return config.getDatabaseName(); + } + + public String getTMode() + { + return config.getTMode(); + } + + @Override + public void close() + { + try { + dropTestDatabaseIfExists(); + if (!connection.isClosed()) { + connection.close(); + } + if (clearScapeSetup != null) { + clearScapeSetup.cleanup(); + } + } + catch (SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean supportsMultiRowInsert() + { + return false; + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/BaseException.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/BaseException.java new file mode 100644 index 000000000000..3eceeaf9c586 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/BaseException.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public class BaseException + extends RuntimeException +{ + private final int statusCode; + + public BaseException(int statusCode, String body) + { + super(body); + this.statusCode = statusCode; + } + + public int getStatusCode() + { + return statusCode; + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeEnvironmentUtils.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeEnvironmentUtils.java new file mode 100644 index 000000000000..3b9f45b3da9a --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeEnvironmentUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +import static java.util.Locale.ENGLISH; + +public final class ClearScapeEnvironmentUtils +{ + private static final String PREFIX = "trino-test-"; + private static final int MAX_ENV_NAME_LENGTH = 30; // Adjust based on ClearScape limits + + private ClearScapeEnvironmentUtils() + { + } + + public static String generateUniqueEnvName(Class testClass) + { + String className = testClass.getSimpleName().toLowerCase(ENGLISH); + String envName = PREFIX + className; + // Truncate if too long + if (envName.length() > MAX_ENV_NAME_LENGTH) { + envName = envName.substring(0, MAX_ENV_NAME_LENGTH); + } + return envName; + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeManager.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeManager.java new file mode 100644 index 000000000000..5ad61678f534 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeManager.java @@ -0,0 +1,143 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +import io.airlift.log.Logger; +import io.trino.plugin.integration.util.TeradataTestConstants; + +import java.net.URISyntaxException; +import java.util.regex.Pattern; + +public class ClearScapeManager +{ + private static final Logger log = Logger.get(ClearScapeManager.class); + private static final Pattern ALLOWED_URL_PATTERN = + Pattern.compile("^(https?://)(www\\.)?api.clearscape.teradata\\.com.*"); + private Model model; + + private boolean isValidUrl(String url) + { + return ALLOWED_URL_PATTERN.matcher(url).matches(); + } + + private TeradataHttpClient getTeradataHttpClient() + throws URISyntaxException + { + String envUrl = TeradataTestConstants.ENV_CLEARSCAPE_URL; + if (isValidUrl(envUrl)) { + return new TeradataHttpClient(envUrl); + } + else { + throw new URISyntaxException(envUrl, "Provide valid environment URL"); + } + } + + public void init(Model model) + { + this.model = model; + } + + public void setup() + { + createAndStartClearScapeInstance(); + } + + public void stop() + { + stopClearScapeInstance(); + } + + public void teardown() + { + shutdownAndDestroyClearScapeInstance(); + } + + private void createAndStartClearScapeInstance() + { + try { + TeradataHttpClient teradataHttpClient = getTeradataHttpClient(); + + String token = this.model.getToken(); + String name = this.model.getEnvName(); + EnvironmentResponse response = null; + try { + response = teradataHttpClient.getEnvironment(new GetEnvironmentRequest(name), token); + } + catch (BaseException be) { + log.info("Environment %s is not available. %s", name, be.getMessage()); + } + + if (response == null || response.ip() == null) { + CreateEnvironmentRequest request = new CreateEnvironmentRequest( + name, + model.getRegion(), + model.getPassword()); + response = teradataHttpClient.createEnvironment(request, token).get(); + } + else if (response.state() == EnvironmentResponse.State.STOPPED) { + EnvironmentRequest request = new EnvironmentRequest(name, new OperationRequest("start")); + teradataHttpClient.startEnvironment(request, token); + } + if (response != null) { + model.setHostName(response.ip()); + } + } + catch (Exception e) { + throw new RuntimeException("Failed to create and start ClearScape instance", e); + } + } + + private void stopClearScapeInstance() + { + try { + TeradataHttpClient teradataHttpClient = getTeradataHttpClient(); + String token = this.model.getToken(); + String name = this.model.getEnvName(); + + EnvironmentResponse response = null; + try { + response = teradataHttpClient.getEnvironment(new GetEnvironmentRequest(name), token); + } + catch (BaseException be) { + log.info("Environment %s is not available. %s", name, be.getMessage()); + } + if (response != null && + response.ip() != null && + response.state() == EnvironmentResponse.State.RUNNING) { + EnvironmentRequest request = new EnvironmentRequest(name, new OperationRequest("stop")); + teradataHttpClient.stopEnvironment(request, token); + } + } + catch (Exception e) { + throw new RuntimeException("Failed to stop ClearScape instance", e); + } + } + + private void shutdownAndDestroyClearScapeInstance() + { + try { + TeradataHttpClient teradataHttpClient = getTeradataHttpClient(); + String token = this.model.getToken(); + DeleteEnvironmentRequest request = new DeleteEnvironmentRequest(this.model.getEnvName()); + teradataHttpClient.deleteEnvironment(request, token).get(); + } + catch (BaseException be) { + log.info("Environment %s is not available. Error - %s", + this.model.getEnvName(), be.getMessage()); + } + catch (Exception e) { + throw new RuntimeException("Failed to shutdown and destroy ClearScape instance", e); + } + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeSetup.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeSetup.java new file mode 100644 index 000000000000..6e621a3c8091 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/ClearScapeSetup.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +import io.trino.plugin.integration.util.TeradataTestConstants; + +import static java.util.Objects.requireNonNull; + +public class ClearScapeSetup +{ + private final String token; + private final String password; + private final String envName; + private final String region; + private final boolean destroyEnv; + private ClearScapeManager manager; + + public ClearScapeSetup( + String token, + String password, + String envName, + boolean destroyEnv, + String region) + { + requireNonNull(token, "token is null"); + requireNonNull(password, "password is null"); + requireNonNull(envName, "envName is null"); + requireNonNull(region, "region is null"); + this.token = token; + this.password = password; + this.envName = envName; + this.region = region; + this.destroyEnv = destroyEnv; + } + + public Model initialize() + { + try { + manager = new ClearScapeManager(); + Model model = createModel(); + manager.init(model); + manager.setup(); + return model; + } + catch (Exception e) { + throw new RuntimeException("Failed to initialize ClearScape environment: " + envName, e); + } + } + + private Model createModel() + { + Model model = new Model(); + model.setEnvName(envName); + model.setUserName(TeradataTestConstants.ENV_CLEARSCAPE_USERNAME); + model.setPassword(password); + model.setDatabaseName(TeradataTestConstants.ENV_CLEARSCAPE_USERNAME); + model.setToken(token); + model.setRegion(region); + return model; + } + + public void cleanup() + { + if (manager == null) { + return; + } + if (destroyEnv) { + manager.teardown(); + return; + } + manager.stop(); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/CreateEnvironmentRequest.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/CreateEnvironmentRequest.java new file mode 100644 index 000000000000..553d0ca1338e --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/CreateEnvironmentRequest.java @@ -0,0 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public record CreateEnvironmentRequest( + String name, + String region, + String password +) {} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/DeleteEnvironmentRequest.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/DeleteEnvironmentRequest.java new file mode 100644 index 000000000000..2bbf60e69b42 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/DeleteEnvironmentRequest.java @@ -0,0 +1,18 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public record DeleteEnvironmentRequest( + String name +) {} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/EnvironmentRequest.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/EnvironmentRequest.java new file mode 100644 index 000000000000..76bb75e1d2ad --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/EnvironmentRequest.java @@ -0,0 +1,19 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public record EnvironmentRequest( + String name, + OperationRequest request +) {} diff --git a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/CatalogNameModule.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/EnvironmentResponse.java similarity index 55% rename from lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/CatalogNameModule.java rename to plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/EnvironmentResponse.java index 1f055819a2e3..d2caa26ec689 100644 --- a/lib/trino-plugin-toolkit/src/main/java/io/trino/plugin/base/CatalogNameModule.java +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/EnvironmentResponse.java @@ -11,27 +11,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.plugin.base; - -import com.google.inject.Binder; -import com.google.inject.Module; -import io.trino.spi.catalog.CatalogName; +package io.trino.plugin.integration.clearscape; +import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; -public class CatalogNameModule - implements Module +public record EnvironmentResponse( + State state, + String region, + String name, + String ip) { - private final String catalogName; - - public CatalogNameModule(String catalogName) - { - this.catalogName = requireNonNull(catalogName, "catalogName is null"); + public EnvironmentResponse { + requireNonNull(state, "state must not be null"); + requireNonNull(region, "name must not be null"); + requireNonNull(name, "name must not be null"); + region = region.toUpperCase(ENGLISH); } - @Override - public void configure(Binder binder) + public enum State { - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); + RUNNING, + STOPPED, } } diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Error4xxException.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Error4xxException.java new file mode 100644 index 000000000000..b32f16e48564 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Error4xxException.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public class Error4xxException + extends BaseException +{ + public Error4xxException(int statusCode, String body) + { + super(statusCode, body); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Error5xxException.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Error5xxException.java new file mode 100644 index 000000000000..355e1db480dd --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Error5xxException.java @@ -0,0 +1,23 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public class Error5xxException + extends BaseException +{ + public Error5xxException(int statusCode, String body) + { + super(statusCode, body); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/GetEnvironmentRequest.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/GetEnvironmentRequest.java new file mode 100644 index 000000000000..04a75c64e9f9 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/GetEnvironmentRequest.java @@ -0,0 +1,18 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public record GetEnvironmentRequest( + String name +) {} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Headers.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Headers.java new file mode 100644 index 000000000000..dadfeac0c5a0 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Headers.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public final class Headers +{ + public static final String CONTENT_TYPE = "Content-Type"; + public static final String AUTHORIZATION = "Authorization"; + public static final String APPLICATION_JSON = "application/json"; + public static final String BEARER = "Bearer "; + + private Headers() + {} +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Model.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Model.java new file mode 100644 index 000000000000..7258ed8e435e --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Model.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public class Model +{ + String envName; + String hostName; + String userName; + String password; + String databaseName; + String token; + String region; + + public String getEnvName() + { + return envName; + } + + public void setEnvName(String envName) + { + this.envName = envName; + } + + public String getHostName() + { + return hostName; + } + + public void setHostName(String hostName) + { + this.hostName = hostName; + } + + public void setUserName(String userName) + { + this.userName = userName; + } + + public String getPassword() + { + return password; + } + + public void setPassword(String password) + { + this.password = password; + } + + public void setDatabaseName(String databaseName) + { + this.databaseName = databaseName; + } + + public String getToken() + { + return token; + } + + public void setToken(String token) + { + this.token = token; + } + + public String getRegion() + { + return region; + } + + public void setRegion(String region) + { + this.region = region; + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/OperationRequest.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/OperationRequest.java new file mode 100644 index 000000000000..a8724424ceee --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/OperationRequest.java @@ -0,0 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +public record OperationRequest( + String operation) {} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Region.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Region.java new file mode 100644 index 000000000000..af490275ea26 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/Region.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +import java.util.Arrays; + +public enum Region { + US_CENTRAL("us-central"), + US_EAST("us-east"), + US_WEST("us-west"), + SOUTHAMERICA_EAST("southamerica-east"), + EUROPE_WEST("europe-west"), + ASIA_SOUTH("asia-south"), + ASIA_NORTHEAST("asia-northeast"), + ASIA_SOUTHEAST("asia-southeast"), + AUSTRALIA_SOUTHEAST("australia-southeast"); + + private final String code; + + Region(String code) + { + this.code = code; + } + + public static boolean isValid(String value) + { + if (value == null) { + return false; + } + return Arrays.stream(Region.values()) + .anyMatch(r -> r.code.equalsIgnoreCase(value)); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/TeradataHttpClient.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/TeradataHttpClient.java new file mode 100644 index 000000000000..ae93d199c9c9 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/clearscape/TeradataHttpClient.java @@ -0,0 +1,171 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.clearscape; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.concurrent.CompletableFuture; + +import static io.trino.plugin.integration.clearscape.Headers.APPLICATION_JSON; +import static io.trino.plugin.integration.clearscape.Headers.AUTHORIZATION; +import static io.trino.plugin.integration.clearscape.Headers.BEARER; +import static io.trino.plugin.integration.clearscape.Headers.CONTENT_TYPE; + +public class TeradataHttpClient +{ + private final String baseUrl; + private final HttpClient httpClient; + private final ObjectMapper objectMapper; + + public TeradataHttpClient(String baseUrl) + { + this(HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(), baseUrl); + } + + public TeradataHttpClient( + HttpClient httpClient, + String baseUrl) + { + this.httpClient = httpClient; + this.baseUrl = baseUrl; + this.objectMapper = JsonMapper.builder() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS, false) + .build(); + } + + // Creating an environment is a blocking operation by default, and it takes ~1.5min to finish + public CompletableFuture createEnvironment(CreateEnvironmentRequest createEnvironmentRequest, + String token) + { + var requestBody = handleCheckedException(() -> objectMapper.writeValueAsString(createEnvironmentRequest)); + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl.concat("/environments"))) + .headers( + AUTHORIZATION, BEARER + token, + CONTENT_TYPE, APPLICATION_JSON) + .POST(HttpRequest.BodyPublishers.ofString(requestBody)) + .build(); + return httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString()) + .thenApply(httpResponse -> handleHttpResponse(httpResponse, new TypeReference<>() {})); + } + + public EnvironmentResponse getEnvironment(GetEnvironmentRequest getEnvironmentRequest, String token) + { + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl + .concat("/environments/") + .concat(getEnvironmentRequest.name()))) + .headers(AUTHORIZATION, BEARER + token) + .GET() + .build(); + var httpResponse = + handleCheckedException(() -> httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString())); + return handleHttpResponse(httpResponse, new TypeReference<>() {}); + } + + // Deleting an environment is a blocking operation by default, and it takes ~1.5min to finish + public CompletableFuture deleteEnvironment(DeleteEnvironmentRequest deleteEnvironmentRequest, String token) + { + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl + .concat("/environments/") + .concat(deleteEnvironmentRequest.name()))) + .headers(AUTHORIZATION, BEARER + token) + .DELETE() + .build(); + + return httpClient.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofString()) + .thenApply(httpResponse -> handleHttpResponse(httpResponse, new TypeReference<>() {})); + } + + public void startEnvironment(EnvironmentRequest environmentRequest, String token) + { + var requestBody = handleCheckedException(() -> objectMapper.writeValueAsString(environmentRequest.request())); + getVoidCompletableFuture(environmentRequest.name(), token, requestBody); + } + + public void stopEnvironment(EnvironmentRequest environmentRequest, String token) + { + var requestBody = handleCheckedException(() -> objectMapper.writeValueAsString(environmentRequest.request())); + getVoidCompletableFuture(environmentRequest.name(), token, requestBody); + } + + private void getVoidCompletableFuture(String name, String token, String jsonPayLoadString) + { + HttpRequest.BodyPublisher publisher = HttpRequest.BodyPublishers.ofString(jsonPayLoadString); + var httpRequest = HttpRequest.newBuilder(URI.create(baseUrl + .concat("/environments/") + .concat(name))) + .headers(AUTHORIZATION, BEARER + token, + CONTENT_TYPE, APPLICATION_JSON) + .method("PATCH", publisher) + .build(); + var httpResponse = + handleCheckedException(() -> httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString())); + handleHttpResponse(httpResponse, new TypeReference<>() {}); + } + + private T handleHttpResponse(HttpResponse httpResponse, TypeReference typeReference) + { + var body = httpResponse.body(); + if (httpResponse.statusCode() >= 200 && httpResponse.statusCode() <= 299) { + return handleCheckedException(() -> { + if (typeReference.getType().getTypeName().equals(Void.class.getTypeName())) { + return null; + } + else { + return objectMapper.readValue(body, typeReference); + } + }); + } + else if (httpResponse.statusCode() >= 400 && httpResponse.statusCode() <= 499) { + throw new Error4xxException(httpResponse.statusCode(), body); + } + else if (httpResponse.statusCode() >= 500 && httpResponse.statusCode() <= 599) { + throw new Error5xxException(httpResponse.statusCode(), body); + } + else { + throw new BaseException(httpResponse.statusCode(), body); + } + } + + private T handleCheckedException(CheckedSupplier checkedSupplier) + { + try { + return checkedSupplier.get(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + @FunctionalInterface + private interface CheckedSupplier + { + T get() + throws IOException, InterruptedException; + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/util/TeradataTestConstants.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/util/TeradataTestConstants.java new file mode 100644 index 000000000000..5e1a7af7097f --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/integration/util/TeradataTestConstants.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.integration.util; + +public interface TeradataTestConstants +{ + String ENV_CLEARSCAPE_URL = "https://api.clearscape.teradata.com"; + String ENV_CLEARSCAPE_USERNAME = "demo_user"; + String ENV_CLEARSCAPE_REGION = "asia-south"; +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestLogonMechanism.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestLogonMechanism.java new file mode 100644 index 000000000000..08943c3b3841 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestLogonMechanism.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.unit; + +import io.trino.plugin.teradata.LogonMechanism; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestLogonMechanism +{ + @Test + public void testFromStringValidValues() + { + assertThat(LogonMechanism.fromString("TD2")).isEqualTo(LogonMechanism.TD2); + assertThat(LogonMechanism.fromString("td2")).isEqualTo(LogonMechanism.TD2); + } + + @Test + public void testGetMechanism() + { + assertThat(LogonMechanism.TD2.getMechanism()).isEqualTo("TD2"); + } + + @Test + public void testFromStringInvalidValue() + { + assertThatThrownBy(() -> LogonMechanism.fromString("UNKNOWN")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown logon mechanism"); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataConfig.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataConfig.java new file mode 100644 index 000000000000..c5f3a5c2cc95 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataConfig.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.unit; + +import io.trino.plugin.teradata.TeradataConfig; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestTeradataConfig +{ + @Test + public void testDefaults() + { + TeradataConfig config = new TeradataConfig(); + assertThat(config.getLogMech()).isEqualTo("TD2"); + assertThat(config.getTeradataCaseSensitivity()).isEqualTo(TeradataConfig.TeradataCaseSensitivity.CASE_SENSITIVE); + } + + @Test + public void testSetters() + { + TeradataConfig config = new TeradataConfig() + .setLogMech("TD2") + .setTeradataCaseSensitivity(TeradataConfig.TeradataCaseSensitivity.CASE_INSENSITIVE); + assertThat(config.getLogMech()).isEqualTo("TD2"); + assertThat(config.getTeradataCaseSensitivity()).isEqualTo(TeradataConfig.TeradataCaseSensitivity.CASE_INSENSITIVE); + } + + @Test + public void testTeradataCaseSensitivityEnum() + { + assertThat(TeradataConfig.TeradataCaseSensitivity.valueOf("CASE_INSENSITIVE")) + .isEqualTo(TeradataConfig.TeradataCaseSensitivity.CASE_INSENSITIVE); + assertThat(TeradataConfig.TeradataCaseSensitivity.valueOf("CASE_SENSITIVE")) + .isEqualTo(TeradataConfig.TeradataCaseSensitivity.CASE_SENSITIVE); + assertThat(TeradataConfig.TeradataCaseSensitivity.valueOf("AS_DEFINED")) + .isEqualTo(TeradataConfig.TeradataCaseSensitivity.AS_DEFINED); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataConstants.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataConstants.java new file mode 100644 index 000000000000..9eb651699432 --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataConstants.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.unit; + +import io.trino.plugin.teradata.util.TeradataConstants; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestTeradataConstants +{ + @Test + public void testConstantsDefined() + { + assertThat(TeradataConstants.TERADATA_OBJECT_NAME_LIMIT).isEqualTo(128); + } +} diff --git a/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataPlugin.java b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataPlugin.java new file mode 100644 index 000000000000..78879c36424e --- /dev/null +++ b/plugin/trino-teradata/src/test/java/io/trino/plugin/unit/TestTeradataPlugin.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.unit; + +import io.trino.plugin.jdbc.JdbcConnectorFactory; +import io.trino.plugin.teradata.TeradataPlugin; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.testing.TestingConnectorContext; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static com.google.common.collect.Iterables.getOnlyElement; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestTeradataPlugin +{ + @Test + public void testCreateConnector() + { + TeradataPlugin plugin = new TeradataPlugin(); + ConnectorFactory factory = getOnlyElement(plugin.getConnectorFactories()); + Assertions.assertNotNull(factory); + assertThat(factory).isInstanceOf(JdbcConnectorFactory.class); + factory.create("test", + Map.of( + "connection-url", "jdbc:teradata://test/"), + new TestingConnectorContext()) + .shutdown(); + } +} diff --git a/plugin/trino-thrift-api/pom.xml b/plugin/trino-thrift-api/pom.xml index 3a4914a7d9fc..cd8f36d766a9 100644 --- a/plugin/trino-thrift-api/pom.xml +++ b/plugin/trino-thrift-api/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift-testing-server/pom.xml b/plugin/trino-thrift-testing-server/pom.xml index 6534c601c4bc..6f144c923fcc 100644 --- a/plugin/trino-thrift-testing-server/pom.xml +++ b/plugin/trino-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift/pom.xml b/plugin/trino-thrift/pom.xml index 4c89ef815297..8293e5a0b062 100644 --- a/plugin/trino-thrift/pom.xml +++ b/plugin/trino-thrift/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftConnectorFactory.java b/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftConnectorFactory.java index 2f3e4c227667..242953ac5286 100644 --- a/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftConnectorFactory.java +++ b/plugin/trino-thrift/src/main/java/io/trino/plugin/thrift/ThriftConnectorFactory.java @@ -17,13 +17,12 @@ import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; import io.airlift.drift.transport.netty.client.DriftNettyClientModule; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; -import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; -import io.trino.spi.type.TypeManager; import org.weakref.jmx.guice.MBeanModule; import java.util.Map; @@ -60,15 +59,13 @@ public Connector create(String catalogName, Map config, Connecto new MBeanServerModule(), new ConnectorObjectNameGeneratorModule("io.trino.plugin.thrift", "trino.plugin.thrift"), new DriftNettyClientModule(), - binder -> { - binder.bind(TypeManager.class).toInstance(context.getTypeManager()); - binder.bind(CatalogName.class).toInstance(new CatalogName(catalogName)); - }, + new ConnectorContextModule(catalogName, context), locationModule, new ThriftModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftConnectorTest.java b/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftConnectorTest.java index 5eaad0e939c3..4b5827e51769 100644 --- a/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftConnectorTest.java +++ b/plugin/trino-thrift/src/test/java/io/trino/plugin/thrift/integration/TestThriftConnectorTest.java @@ -39,6 +39,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_VIEW, SUPPORTS_DELETE, SUPPORTS_INSERT, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MERGE, SUPPORTS_NOT_NULL_CONSTRAINT, SUPPORTS_RENAME_COLUMN, diff --git a/plugin/trino-tpcds/pom.xml b/plugin/trino-tpcds/pom.xml index 487fc834bae2..650f7562b462 100644 --- a/plugin/trino-tpcds/pom.xml +++ b/plugin/trino-tpcds/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsConnectorFactory.java b/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsConnectorFactory.java index 6b2b2aeb7fe6..3363b849a4ec 100644 --- a/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsConnectorFactory.java +++ b/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsConnectorFactory.java @@ -15,6 +15,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; import io.trino.spi.connector.ConnectorFactory; @@ -37,10 +38,14 @@ public Connector create(String catalogName, Map config, Connecto { checkStrictSpiVersionMatch(context, this); - Bootstrap app = new Bootstrap("io.trino.bootstrap.catalog." + catalogName, new TpcdsModule(context.getNodeManager())); + Bootstrap app = new Bootstrap( + "io.trino.bootstrap.catalog." + catalogName, + new ConnectorContextModule(catalogName, context), + new TpcdsModule()); Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .initialize(); diff --git a/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsModule.java b/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsModule.java index d3a746601b90..5b18dddc512d 100644 --- a/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsModule.java +++ b/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsModule.java @@ -16,26 +16,16 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.google.inject.Scopes; -import io.trino.spi.NodeManager; import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.util.Objects.requireNonNull; public class TpcdsModule implements Module { - private final NodeManager nodeManager; - - public TpcdsModule(NodeManager nodeManager) - { - this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); - } - @Override public void configure(Binder binder) { configBinder(binder).bindConfig(TpcdsConfig.class); - binder.bind(NodeManager.class).toInstance(nodeManager); binder.bind(TpcdsSessionProperties.class).in(Scopes.SINGLETON); binder.bind(TpcdsMetadata.class).in(Scopes.SINGLETON); binder.bind(TpcdsSplitManager.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsSplit.java b/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsSplit.java index 55055a33aa6c..7343a155b2a2 100644 --- a/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsSplit.java +++ b/plugin/trino-tpcds/src/main/java/io/trino/plugin/tpcds/TpcdsSplit.java @@ -106,8 +106,8 @@ public boolean equals(Object obj) } TpcdsSplit other = (TpcdsSplit) obj; return this.totalParts == other.totalParts && - this.partNumber == other.partNumber && - this.noSexism == other.noSexism; + this.partNumber == other.partNumber && + this.noSexism == other.noSexism; } @Override diff --git a/plugin/trino-tpch/pom.xml b/plugin/trino-tpch/pom.xml index 87fcb2a2991a..2500435c48ec 100644 --- a/plugin/trino-tpch/pom.xml +++ b/plugin/trino-tpch/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchConnectorFactory.java b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchConnectorFactory.java index 15fe4f0b90c8..8ccbcaa354a3 100644 --- a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchConnectorFactory.java +++ b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchConnectorFactory.java @@ -16,7 +16,7 @@ import com.google.inject.Injector; import io.airlift.bootstrap.Bootstrap; import io.airlift.json.JsonModule; -import io.opentelemetry.api.OpenTelemetry; +import io.trino.plugin.base.ConnectorContextModule; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorContext; @@ -62,13 +62,15 @@ public Connector create(String catalogName, Map properties, Conn Bootstrap app = new Bootstrap( "io.trino.bootstrap.catalog." + catalogName, - binder -> binder.bind(OpenTelemetry.class).toInstance(context.getOpenTelemetry()), + new ConnectorContextModule(catalogName, context), new MBeanModule(), new JsonModule(), - new TpchModule(context.getNodeManager(), defaultSplitsPerNode, predicatePushdownEnabled), + new TpchModule(defaultSplitsPerNode, predicatePushdownEnabled), new MBeanServerModule()); - Injector injector = app.doNotInitializeLogging() + Injector injector = app + .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(properties) .initialize(); diff --git a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchModule.java b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchModule.java index 38c7530a5a73..61028f1ffa8f 100644 --- a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchModule.java +++ b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchModule.java @@ -16,7 +16,6 @@ import com.google.inject.Binder; import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.spi.NodeManager; import io.trino.spi.connector.Connector; import io.trino.spi.connector.ConnectorMetadata; import io.trino.spi.connector.ConnectorNodePartitioningProvider; @@ -24,18 +23,15 @@ import io.trino.spi.connector.ConnectorSplitManager; import static io.airlift.configuration.ConfigBinder.configBinder; -import static java.util.Objects.requireNonNull; public class TpchModule extends AbstractConfigurationAwareModule { - private final NodeManager nodeManager; private final int defaultSplitsPerNode; private final boolean predicatePushdownEnabled; - public TpchModule(NodeManager nodeManager, int defaultSplitsPerNode, boolean predicatePushdownEnabled) + public TpchModule(int defaultSplitsPerNode, boolean predicatePushdownEnabled) { - this.nodeManager = requireNonNull(nodeManager, "nodeManager is null"); this.defaultSplitsPerNode = defaultSplitsPerNode; this.predicatePushdownEnabled = predicatePushdownEnabled; } @@ -43,7 +39,6 @@ public TpchModule(NodeManager nodeManager, int defaultSplitsPerNode, boolean pre @Override protected void setup(Binder binder) { - binder.bind(NodeManager.class).toInstance(nodeManager); binder.bind(Connector.class).to(TpchConnector.class).in(Scopes.SINGLETON); binder.bind(ConnectorMetadata.class).to(TpchMetadata.class).in(Scopes.SINGLETON); binder.bind(ConnectorPageSourceProvider.class).to(TpchPageSourceProvider.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchSplit.java b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchSplit.java index bb97b2967b51..40e644fa0238 100644 --- a/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchSplit.java +++ b/plugin/trino-tpch/src/main/java/io/trino/plugin/tpch/TpchSplit.java @@ -89,7 +89,7 @@ public boolean equals(Object obj) } TpchSplit other = (TpchSplit) obj; return this.totalParts == other.totalParts && - this.partNumber == other.partNumber; + this.partNumber == other.partNumber; } @Override diff --git a/plugin/trino-vertica/pom.xml b/plugin/trino-vertica/pom.xml index 61243d0cf745..438367b8d88f 100644 --- a/plugin/trino-vertica/pom.xml +++ b/plugin/trino-vertica/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -256,13 +256,13 @@ org.testcontainers - jdbc + testcontainers test org.testcontainers - testcontainers + testcontainers-jdbc test diff --git a/pom.xml b/pom.xml index 7286768fe842..375b99c4400d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,12 +5,12 @@ io.airlift airbase - 306 + 324 io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT pom ${project.artifactId} @@ -113,6 +113,7 @@ plugin/trino-snowflake plugin/trino-spooling-filesystem plugin/trino-sqlserver + plugin/trino-teradata plugin/trino-teradata-functions plugin/trino-thrift plugin/trino-thrift-api @@ -150,7 +151,7 @@ 24 - 2025-09-24T23:09:23Z + 2025-10-29T18:00:28Z ERROR @@ -182,34 +183,33 @@ ${air.test.jvm.additional-arguments.default} - 363 + 372 2.9.6 4.13.2 - 1.12.0 - 1.12.791 + 1.12.1 + 1.12.793 4.17.0 - 10.23.1 7.8.0 119 1.24 - 11.13.1 - 1.15.1 + 11.15.0 + 1.15.4 v22.14.0 11.2.0 - 4.5.4 + 4.5.6 5.3.6 1.10.0 - 5.18.0 + 5.18.1 0.13.0 1.20.0 4.1.0 - 5.1.0 - 3.16.0 + 5.3.0 + 3.16.2 1.16.0 ${dep.plugin.surefire.version} 3.25.8 - 3.26.1 - 2.2.37 + 3.27.1 + 2.2.40 2.3.2 2.0.74.Final 202 @@ -229,7 +229,7 @@ com.azure azure-sdk-bom - 1.2.38 + 1.3.0 pom import @@ -237,7 +237,7 @@ com.google.cloud libraries-bom - 26.68.0 + 26.71.0 pom import @@ -245,7 +245,7 @@ com.oracle.database.jdbc ojdbc11-production - 23.9.0.25.07 + 23.26.0.0.0 pom import @@ -269,7 +269,7 @@ io.netty netty-bom - 4.2.6.Final + 4.2.7.Final pom import @@ -285,7 +285,7 @@ org.jdbi jdbi3-bom - 3.49.5 + 3.49.6 pom import @@ -293,7 +293,7 @@ org.testcontainers testcontainers-bom - 1.21.3 + 2.0.1 pom import @@ -301,7 +301,7 @@ software.amazon.awssdk bom - 2.34.3 + 2.37.3 pom import @@ -363,7 +363,7 @@ com.azure azure-core-tracing-opentelemetry - 1.0.0-beta.60 + 1.0.0-beta.61 com.azure @@ -418,7 +418,7 @@ com.github.ben-manes.caffeine caffeine - 3.2.2 + 3.2.3 @@ -430,13 +430,13 @@ com.github.luben zstd-jni - 1.5.7-4 + 1.5.7-6 com.github.oshi oshi-core - 6.9.0 + 6.9.1 @@ -448,7 +448,7 @@ com.google.cloud.bigdataoss gcs-connector - 3.1.7 + 3.1.9 shaded @@ -504,13 +504,13 @@ com.microsoft.sqlserver mssql-jdbc - 13.2.0.jre11 + 13.2.1.jre11 com.mysql mysql-connector-j - 9.2.0 + 9.5.0 com.google.protobuf @@ -528,8 +528,7 @@ com.nimbusds oauth2-oidc-sdk - 11.29.1 - jdk11 + 11.30 @@ -644,7 +643,7 @@ io.airlift units - 1.10 + 1.12 @@ -790,6 +789,12 @@ 4.2.37 + + io.github.jeschkies + loki-client + 0.0.5 + + io.jsonwebtoken jjwt-api @@ -811,7 +816,7 @@ io.minio minio - 8.5.17 + 8.6.0 com.github.spotbugs @@ -909,13 +914,13 @@ io.projectreactor reactor-core - 3.7.11 + 3.7.12 io.projectreactor.netty reactor-netty-core - 1.2.10 + 1.2.11 @@ -1686,7 +1691,7 @@ it.unimi.dsi fastutil - 8.5.16 + 8.5.18 @@ -1783,6 +1788,14 @@ io.etcd jetcd-core + + io.prometheus + prometheus-metrics-exporter-servlet-jakarta + + + io.prometheus + prometheus-metrics-instrumentation-jvm + io.swagger swagger-annotations @@ -1939,7 +1952,7 @@ org.apache.httpcomponents.client5 httpclient5 - 5.5 + 5.5.1 @@ -2221,7 +2234,7 @@ org.checkerframework checker-qual - 3.51.0 + 3.51.1 @@ -2479,7 +2492,7 @@ org.codehaus.mojo exec-maven-plugin - 3.5.1 + 3.6.2 @@ -2555,6 +2568,8 @@ javax.inject:javax.inject javax.annotation:javax.annotation-api + + com.amazonaws:* @@ -2646,6 +2661,21 @@ mozilla/public-suffix-list.txt + + + + com.github.docker-java + docker-java-transport-zerodep + + + org.apache.httpcomponents.client5 + httpclient5 + + + + org/publicsuffix/list/effective_tld_names.dat + + @@ -2783,6 +2813,24 @@ + + + + com.nimbusds + oauth2-oidc-sdk + + + net.snowflake + snowflake-jdbc + + + + iso3166_1alpha-2-3-map.properties + iso3166_1alpha2-codes.properties + iso3166_1alpha3-codes.properties + iso3166_3-codes.properties + + @@ -2793,7 +2841,7 @@ io.trino trino-maven-plugin - 15 + 17 true @@ -2968,6 +3016,7 @@ -Xep:PreferredInterfaceType:OFF \ -Xep:PrimitiveArrayPassedToVarargsMethod:ERROR \ + -Xep:RedundantNullCheck:ERROR \ -Xep:RethrowReflectiveOperationExceptionAsLinkageError:OFF \ -Xep:StaticAssignmentOfThrowable:ERROR \ -Xep:StaticGuardedByInstance:ERROR \ diff --git a/service/trino-proxy/pom.xml b/service/trino-proxy/pom.xml index b551ee961529..64ce4ec366e6 100644 --- a/service/trino-proxy/pom.xml +++ b/service/trino-proxy/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/service/trino-verifier/README.md b/service/trino-verifier/README.md index f23a6d655cb9..31e893d5b7f6 100644 --- a/service/trino-verifier/README.md +++ b/service/trino-verifier/README.md @@ -43,13 +43,12 @@ test.gateway=jdbc:trino://localhost:8081 thread-count=1 ``` -Lastly, download the [Maven verifier plugin][maven_download] for the same -release as your Trino instance by navigating to the directory for that +Lastly, download the [Maven verifier +plugin](https://repo.maven.apache.org/maven2/io/trino/trino-verifier/) for the +same release as your Trino instance by navigating to the directory for that release, and selecting the ``trino-verifier-*.jar`` file. Once it is downloaded, rename it to `verifier`, make it executable with `chmod +x`, then run it: -[maven_download]: https://repo.maven.apache.org/maven2/io/trino/trino-verifier/ - ``` ./verifier config.properties ``` diff --git a/service/trino-verifier/pom.xml b/service/trino-verifier/pom.xml index e9b8ea8b2f75..6dc5f2aeffe3 100644 --- a/service/trino-verifier/pom.xml +++ b/service/trino-verifier/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -160,7 +160,7 @@ org.testcontainers - mysql + testcontainers-mysql test diff --git a/service/trino-verifier/src/test/java/io/trino/verifier/TestDatabaseEventClient.java b/service/trino-verifier/src/test/java/io/trino/verifier/TestDatabaseEventClient.java index 7e277b1c9499..ffc49523da9b 100644 --- a/service/trino-verifier/src/test/java/io/trino/verifier/TestDatabaseEventClient.java +++ b/service/trino-verifier/src/test/java/io/trino/verifier/TestDatabaseEventClient.java @@ -22,7 +22,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.parallel.Execution; -import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.mysql.MySQLContainer; import java.sql.Connection; import java.sql.DriverManager; @@ -96,7 +96,7 @@ public class TestDatabaseEventClient null, null); - private MySQLContainer mysqlContainer; + private MySQLContainer mysqlContainer; private String mysqlContainerUrl; private JsonCodec> codec; private DatabaseEventClient eventClient; @@ -104,7 +104,7 @@ public class TestDatabaseEventClient @BeforeAll public void setup() { - mysqlContainer = new MySQLContainer<>("mysql:8.0.36"); + mysqlContainer = new MySQLContainer("mysql:8.0.36"); mysqlContainer.start(); mysqlContainerUrl = getJdbcUrl(mysqlContainer); codec = new JsonCodecFactory().listJsonCodec(String.class); @@ -115,7 +115,7 @@ public void setup() eventClient.postConstruct(); } - private static String getJdbcUrl(MySQLContainer container) + private static String getJdbcUrl(MySQLContainer container) { return format("%s?user=%s&password=%s&useSSL=false&allowPublicKeyRetrieval=true", container.getJdbcUrl(), diff --git a/testing/trino-benchmark-queries/pom.xml b/testing/trino-benchmark-queries/pom.xml index e723150e7743..a606b865436f 100644 --- a/testing/trino-benchmark-queries/pom.xml +++ b/testing/trino-benchmark-queries/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-benchto-benchmarks/pom.xml b/testing/trino-benchto-benchmarks/pom.xml index 3c350841dfc6..205532e9d9b1 100644 --- a/testing/trino-benchto-benchmarks/pom.xml +++ b/testing/trino-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-faulttolerant-tests/pom.xml b/testing/trino-faulttolerant-tests/pom.xml index 3b67d39f30e7..7ae30cf6ca44 100644 --- a/testing/trino-faulttolerant-tests/pom.xml +++ b/testing/trino-faulttolerant-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -390,31 +390,31 @@ org.testcontainers - mongodb + testcontainers test org.testcontainers - mssqlserver + testcontainers-mongodb test org.testcontainers - mysql + testcontainers-mssqlserver test org.testcontainers - postgresql + testcontainers-mysql test org.testcontainers - testcontainers + testcontainers-postgresql test @@ -444,6 +444,20 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + org.apache.maven.plugins maven-surefire-plugin diff --git a/testing/trino-plugin-reader/pom.xml b/testing/trino-plugin-reader/pom.xml index 575902599c8d..53a8f51012b4 100644 --- a/testing/trino-plugin-reader/pom.xml +++ b/testing/trino-plugin-reader/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginLoader.java b/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginLoader.java index 0820fb2d79bc..c07907ba6d41 100644 --- a/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginLoader.java +++ b/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginLoader.java @@ -17,7 +17,7 @@ import io.trino.spi.Plugin; import io.trino.spi.classloader.ThreadContextClassLoader; -import java.io.File; +import java.nio.file.Path; import java.util.List; import java.util.ServiceLoader; import java.util.function.Supplier; @@ -67,10 +67,10 @@ public static void printPluginFeatures(Plugin plugin) plugin.getExchangeManagerFactories().forEach(factory -> System.out.println(EXCHANGE_MANAGER + factory.getName())); } - public static List loadPlugins(File path) + public static List loadPlugins(List path) { ServerPluginsProviderConfig config = new ServerPluginsProviderConfig(); - config.setInstalledPluginsDir(path); + config.setInstalledPluginsDirs(path); ServerPluginsProvider pluginsProvider = new ServerPluginsProvider(config, directExecutor()); ImmutableList.Builder plugins = ImmutableList.builder(); pluginsProvider.loadPlugins((plugin, createClassLoader) -> loadPlugin(createClassLoader, plugins), PluginManager::createClassLoader); diff --git a/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginReader.java b/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginReader.java index f04b2faebd5b..34ddcd7e5ec8 100644 --- a/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginReader.java +++ b/testing/trino-plugin-reader/src/main/java/io/trino/server/PluginReader.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Map; import java.util.Optional; @@ -45,8 +46,8 @@ public class PluginReader @Option(names = {"-i", "--impacted-modules"}, description = "Impacted modules file generated by the gitflow-incremental-builder (GIB) Maven plugin") private Optional impactedModulesFile; - @Option(names = {"-p", "--plugin-dir"}, description = "Trino plugin directory") - private File pluginDir = new File("plugin"); + @Option(names = {"-p", "--plugin-dir"}, description = "Trino plugin directories", arity = "1..*") + private List pluginDirs = List.of(Path.of("plugin")); @Option(names = {"-r", "--root-pom"}, description = "Trino root module pom.xml") private File rootPom = new File("pom.xml"); @@ -82,7 +83,7 @@ public Integer call() } } - Map plugins = loadPlugins(pluginDir).stream() + Map plugins = loadPlugins(pluginDirs).stream() .collect(toMap(plugin -> plugin.getClass().getName(), identity())); modulesStream.forEach(entry -> { if (!plugins.containsKey(entry.getValue())) { diff --git a/testing/trino-plugin-reader/src/test/java/io/trino/server/TestPluginReader.java b/testing/trino-plugin-reader/src/test/java/io/trino/server/TestPluginReader.java index 14843ddb61cf..0c20d08a349c 100644 --- a/testing/trino-plugin-reader/src/test/java/io/trino/server/TestPluginReader.java +++ b/testing/trino-plugin-reader/src/test/java/io/trino/server/TestPluginReader.java @@ -14,10 +14,12 @@ package io.trino.server; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import picocli.CommandLine; import java.io.PrintWriter; import java.io.StringWriter; +import java.nio.file.Path; import static org.assertj.core.api.Assertions.assertThat; @@ -39,4 +41,22 @@ void testCall() assertThat(exitCode).isEqualTo(0); assertThat(writer.toString()).isEqualTo(""); } + + @Test + void testCallMultiplePluginDirs(@TempDir Path tempDir) + { + PluginReader pluginReader = new PluginReader(); + StringWriter writer = new StringWriter(); + CommandLine cmd = new CommandLine(pluginReader) + .setOut(new PrintWriter(writer)) + .setErr(new PrintWriter(writer)); + + int exitCode = cmd.execute( + "--impacted-modules", "src/test/resources/gib-impacted.log", + "--plugin-dir", "src/test/resources/server-plugins", + "--plugin-dir", tempDir.toString(), + "--root-pom", "src/test/resources/pom.xml"); + assertThat(exitCode).isEqualTo(0); + assertThat(writer.toString()).isEqualTo(""); + } } diff --git a/testing/trino-product-tests-groups/pom.xml b/testing/trino-product-tests-groups/pom.xml index fbb3d8a399e0..338ca34e7473 100644 --- a/testing/trino-product-tests-groups/pom.xml +++ b/testing/trino-product-tests-groups/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java index cb81b05109b3..06ec00ebac65 100644 --- a/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java +++ b/testing/trino-product-tests-groups/src/main/java/io/trino/tests/product/TestGroups.java @@ -50,6 +50,7 @@ public final class TestGroups public static final String HDFS_IMPERSONATION = "hdfs_impersonation"; public static final String HDFS_NO_IMPERSONATION = "hdfs_no_impersonation"; public static final String HIVE_GCS = "hive_gcs"; + public static final String HIVE4 = "hive4"; public static final String HIVE_SPARK = "hive_spark"; public static final String HIVE_SPARK_NO_STATS_FALLBACK = "hive_spark_no_stats_fallback"; public static final String HIVE_COMPRESSION = "hive_compression"; @@ -71,6 +72,7 @@ public final class TestGroups public static final String LDAP_CLI = "ldap_cli"; public static final String LDAP_AND_FILE_CLI = "ldap_and_file_cli"; public static final String LDAP_MULTIPLE_BINDS = "ldap_multiple_binds"; + public static final String LOKI = "loki"; public static final String TLS = "tls"; public static final String LARGE_QUERY = "large_query"; public static final String KAFKA = "kafka"; @@ -94,11 +96,9 @@ public final class TestGroups public static final String DELTA_LAKE_AZURE = "delta-lake-azure"; public static final String DELTA_LAKE_GCS = "delta-lake-gcs"; public static final String DELTA_LAKE_DATABRICKS = "delta-lake-databricks"; - public static final String DELTA_LAKE_DATABRICKS_122 = "delta-lake-databricks-122"; public static final String DELTA_LAKE_DATABRICKS_133 = "delta-lake-databricks-133"; public static final String DELTA_LAKE_DATABRICKS_143 = "delta-lake-databricks-143"; public static final String DELTA_LAKE_DATABRICKS_154 = "delta-lake-databricks-154"; - public static final String DELTA_LAKE_EXCLUDE_113 = "delta-lake-exclude-113"; // TODO: Remove it once we support generatedColumns, particularly for writes in Delta Lake public static final String DELTA_LAKE_EXCLUDE_164 = "delta-lake-exclude-164"; public static final String DELTA_LAKE_ALLUXIO_CACHING = "delta-lake-alluxio-caching"; diff --git a/testing/trino-product-tests-launcher/bin/run-launcher b/testing/trino-product-tests-launcher/bin/run-launcher index a4e2a2f9fc9c..b35981cd686b 100755 --- a/testing/trino-product-tests-launcher/bin/run-launcher +++ b/testing/trino-product-tests-launcher/bin/run-launcher @@ -9,7 +9,7 @@ if command -v mvnd >/dev/null; then trino_version=$(mvnd -B help:evaluate -Dexpression=pom.version -q -DforceStdout --raw-streams -Dmvnd.logPurgePeriod=999999d) mvn="mvnd" else - trino_version=$(./mvnw -B help:evaluate -Dexpression=pom.version -q -DforceStdout) + trino_version=$(./mvnw -B help:evaluate -Dexpression=pom.version -q -DforceStdout --raw-streams) mvn="./mvnw" fi launcher_jar="${target}/trino-product-tests-launcher-${trino_version}-executable.jar" diff --git a/testing/trino-product-tests-launcher/pom.xml b/testing/trino-product-tests-launcher/pom.xml index fd6d7916f0a7..cb28a6abebc8 100644 --- a/testing/trino-product-tests-launcher/pom.xml +++ b/testing/trino-product-tests-launcher/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -138,12 +138,6 @@ jakarta.annotation-api - - junit - junit - 4.13.2 - - org.apache.commons commons-compress @@ -258,7 +252,7 @@ org.apache.hive hive-jdbc standalone - 3.1.3 + 4.1.0 jar ${project.build.directory} hive-jdbc.jar @@ -266,7 +260,7 @@ com.databricks databricks-jdbc - 2.6.36 + 3.0.3 jar ${project.build.directory} databricks-jdbc.jar diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/Launcher.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/Launcher.java index 2c276159638c..8b8452f5be00 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/Launcher.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/cli/Launcher.java @@ -179,8 +179,7 @@ protected Object[][] getContents() {"server.package", "core/trino-server/target/trino-server-" + getProjectVersion() + ".tar.gz"}, {"launcher.bin", "testing/trino-product-tests-launcher/bin/run-launcher"}, {"cli.bin", format("client/trino-cli/target/trino-cli-%s-executable.jar", getProjectVersion())}, - {"jdk.current", Files.readString(jdkDistribution).trim()}, - {"jdk.distributions", jdkDistribution.getParent().toAbsolutePath().toString()} + {"jdk.current.release", Files.readString(jdkDistribution).trim()}, }; } catch (IOException e) { @@ -190,7 +189,7 @@ protected Object[][] getContents() protected Path findJdkDistribution() { - String searchFor = "core/jdk/current"; + String searchFor = "core/.temurin-release"; Path currentWorkingDirectory = Paths.get("").toAbsolutePath(); Path current = currentWorkingDirectory; // current working directory diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java index aeb25c883ad1..b8bc34b57700 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/DockerContainer.java @@ -157,7 +157,6 @@ public void copyFileToContainer(Transferable transferable, String containerPath) public DockerContainer withExposedLogPaths(String... logPaths) { - requireNonNull(this.logPaths, "log paths are already exposed"); this.logPaths.addAll(Arrays.asList(logPaths)); return this; } @@ -292,6 +291,10 @@ public ExecResult execCommandForResult(String... command) public void copyLogsToHostPath(Path hostPath) { + if (logPaths.isEmpty()) { + return; + } + if (!isRunning()) { log.warn("Could not copy files from stopped container %s", logicalName); return; diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java index 9adf421d384a..c37ec10e6b03 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentModule.java @@ -22,6 +22,7 @@ import io.trino.tests.product.launcher.env.common.HadoopKerberos; import io.trino.tests.product.launcher.env.common.HadoopKerberosKms; import io.trino.tests.product.launcher.env.common.HadoopKerberosKmsWithImpersonation; +import io.trino.tests.product.launcher.env.common.Hive4WithMinio; import io.trino.tests.product.launcher.env.common.HttpProxy; import io.trino.tests.product.launcher.env.common.HttpsProxy; import io.trino.tests.product.launcher.env.common.HydraIdentityProvider; @@ -36,8 +37,8 @@ import io.trino.tests.product.launcher.env.common.StandardMultinode; import io.trino.tests.product.launcher.env.common.TaskRetriesMultinode; import io.trino.tests.product.launcher.env.environment.SpoolingMinio; -import io.trino.tests.product.launcher.env.jdk.DistributionDownloadingJdkProvider; import io.trino.tests.product.launcher.env.jdk.JdkProvider; +import io.trino.tests.product.launcher.env.jdk.TemurinJdkProvider; import io.trino.tests.product.launcher.testcontainers.PortBinder; import java.io.File; @@ -99,6 +100,7 @@ public void configure(Binder binder) binder.bind(OpenLdapReferral.class).in(SINGLETON); binder.bind(HttpProxy.class).in(SINGLETON); binder.bind(HttpsProxy.class).in(SINGLETON); + binder.bind(Hive4WithMinio.class).in(SINGLETON); MapBinder environments = newMapBinder(binder, String.class, EnvironmentProvider.class); findEnvironmentsByBasePackage(ENVIRONMENT_PACKAGE).forEach(clazz -> environments.addBinding(nameForEnvironmentClass(clazz)).to(clazz).in(SINGLETON)); @@ -123,9 +125,9 @@ public EnvironmentConfig provideEnvironmentConfig(EnvironmentOptions options, En @Singleton public JdkProvider provideJdk(Map jdkProviders, EnvironmentOptions options) { - String version = firstNonNull(options.jdkVersion, "").trim().toLowerCase(ENGLISH); + String version = firstNonNull(options.trinoJdkRelease, "").trim().toLowerCase(ENGLISH); if (version.isBlank()) { - throw new IllegalArgumentException("Expected non-empty --trino-jdk-version"); + throw new IllegalArgumentException("Expected non-empty --trino-jdk-release"); } JdkProvider jdkProvider = jdkProviders.get(canonicalJdkProviderName(version)); @@ -133,7 +135,7 @@ public JdkProvider provideJdk(Map jdkProviders, Environment return jdkProvider; } - return new DistributionDownloadingJdkProvider(requireNonNull(options.jdkDistributions, "--trino-jdk-paths is empty"), version, options.jdkDownloadPath); + return new TemurinJdkProvider(version, options.jdkDownloadPath); } @Provides diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java index b37dec6773c1..85c74e1cffe2 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/EnvironmentOptions.java @@ -23,7 +23,6 @@ import static com.google.common.base.Preconditions.checkArgument; import static io.trino.tests.product.launcher.env.EnvironmentContainers.COORDINATOR; -import static io.trino.tests.product.launcher.env.jdk.BuiltInJdkProvider.BUILT_IN_NAME; import static java.util.Locale.ENGLISH; import static picocli.CommandLine.Option; @@ -60,11 +59,8 @@ public final class EnvironmentOptions @Option(names = "--launcher-bin", paramLabel = "", description = "Launcher bin path (used to display run commands)", defaultValue = "${launcher.bin}", hidden = true) public String launcherBin; - @Option(names = "--trino-jdk-version", paramLabel = "", description = "JDK release to run Trino with " + DEFAULT_VALUE, defaultValue = "${jdk.current}") - public String jdkVersion = BUILT_IN_NAME; - - @Option(names = "--trino-jdk-paths", paramLabel = "", description = "Path to JDK distributions " + DEFAULT_VALUE, defaultValue = "${jdk.distributions}") - public String jdkDistributions; + @Option(names = "--trino-jdk-release", paramLabel = "", description = "JDK release to run Trino with " + DEFAULT_VALUE, defaultValue = "${jdk.current.release}") + public String trinoJdkRelease = "${jdk.current.release}"; @Option(names = "--jdk-tmp-download-path", paramLabel = "", defaultValue = "${env:PTL_TMP_DOWNLOAD_PATH:-${sys:java.io.tmpdir}/ptl-tmp-download}", description = "Path to use to download JDK distributions " + DEFAULT_VALUE) @Nullable diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hive4WithMinio.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hive4WithMinio.java new file mode 100644 index 000000000000..4ee02eb93028 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Hive4WithMinio.java @@ -0,0 +1,149 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.common; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.env.DockerContainer; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.testcontainers.PortBinder; +import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.time.Duration; +import java.util.List; +import java.util.Set; + +import static io.trino.tests.product.launcher.docker.ContainerUtil.forSelectedPorts; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.TESTS; +import static io.trino.tests.product.launcher.env.EnvironmentContainers.configureTempto; +import static io.trino.tests.product.launcher.env.common.Minio.MINIO_CONTAINER_NAME; +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +public class Hive4WithMinio + implements EnvironmentExtender +{ + public static final String METASTORE = "metastore"; + public static final String HIVESERVER2 = "hiveserver2"; + private static final int HIVE_SERVER_PORT = 10000; + private static final int HIVE_METASTORE_PORT = 9083; + private static final String APACHE_HIVE_IMAGE = "ghcr.io/trinodb/testing/hive4.0-hive"; + private static final File HIVE_JDBC_PROVIDER = new File("testing/trino-product-tests-launcher/target/hive-jdbc.jar"); + private static final String S3_BUCKET_NAME = "test-bucket"; + + private final PortBinder portBinder; + private final String hadoopImagesVersion; + private final DockerFiles.ResourceProvider configDir; + private final Minio minio; + + @Inject + public Hive4WithMinio( + DockerFiles dockerFiles, + PortBinder portBinder, + EnvironmentConfig config, + Minio minio) + { + this.portBinder = requireNonNull(portBinder, "portBinder is null"); + this.hadoopImagesVersion = requireNonNull(config, "config is null").getHadoopImagesVersion(); + this.configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("common/hive4-with-minio"); + this.minio = requireNonNull(minio, "minio is null"); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + builder.addContainer(createMetastoreServer()); + builder.addContainer(createHiveserver2()); + builder.containerDependsOn(HIVESERVER2, METASTORE); + + configureMinio(builder); + configureTests(builder); + configureTempto(builder, configDir); + } + + @Override + public List getDependencies() + { + return ImmutableList.of(minio); + } + + private DockerContainer createMetastoreServer() + { + DockerContainer container = new DockerContainer(APACHE_HIVE_IMAGE + ":" + hadoopImagesVersion, METASTORE) + .withEnv("SERVICE_NAME", "metastore") + .withCopyFileToContainer( + forHostPath(configDir.getPath("hive-site.xml")), + "/opt/hive/conf/hive-site.xml") + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(Wait.forListeningPort()) + .withStartupTimeout(Duration.ofMinutes(5)); + + portBinder.exposePort(container, HIVE_METASTORE_PORT); + return container; + } + + private DockerContainer createHiveserver2() + { + DockerContainer container = new DockerContainer(APACHE_HIVE_IMAGE + ":" + hadoopImagesVersion, HIVESERVER2) + .withEnv("SERVICE_NAME", "hiveserver2") + .withEnv("SERVICE_OPTS", "-Xmx1G -Dhive.metastore.uris=%s".formatted(URI.create("thrift://%s:%d".formatted(METASTORE, HIVE_METASTORE_PORT)))) + .withEnv("IS_RESUME", "true") + .withEnv("AWS_ACCESS_KEY_ID", "minio-access-key") + .withEnv("AWS_SECRET_KEY", "minio-secret-key") + .withCopyFileToContainer( + forHostPath(configDir.getPath("hive-site.xml")), + "/opt/hive/conf/hive-site.xml") + .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .waitingFor(forSelectedPorts(HIVE_SERVER_PORT)) + .withStartupTimeout(Duration.ofMinutes(5)); + + portBinder.exposePort(container, HIVE_SERVER_PORT); + return container; + } + + private void configureMinio(Environment.Builder builder) + { + FileAttribute> posixFilePermissions = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")); + Path minioBucketDirectory; + try { + minioBucketDirectory = Files.createTempDirectory("test-bucket-contents", posixFilePermissions); + minioBucketDirectory.toFile().deleteOnExit(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + builder.configureContainer(MINIO_CONTAINER_NAME, container -> + container.withCopyFileToContainer(forHostPath(minioBucketDirectory), "/data/" + S3_BUCKET_NAME)); + } + + private void configureTests(Environment.Builder builder) + { + builder.configureContainer(TESTS, dockerContainer -> + dockerContainer + .withEnv("S3_BUCKET", S3_BUCKET_NAME) + .withCopyFileToContainer(forHostPath(HIVE_JDBC_PROVIDER.getAbsolutePath()), "/docker/jdbc/hive-jdbc.jar")); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java index 3af695f4b646..fa7ab8316075 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/HydraIdentityProvider.java @@ -79,7 +79,7 @@ public void extendEnvironment(Environment.Builder builder) .withEnv("OAUTH2_EXPOSE_INTERNAL_ERRORS", "1") .withEnv("GODEBUG", "http2debug=1") .withEnv("DSN", DSN) - .withEnv("URLS_SELF_ISSUER", "http://hydra:4444/") + .withEnv("URLS_SELF_ISSUER", "https://hydra:4444/") .withEnv("URLS_CONSENT", "http://hydra-consent:3000/consent") .withEnv("URLS_LOGIN", "http://hydra-consent:3000/login") .withEnv("SERVE_TLS_KEY_PATH", "/tmp/certs/hydra.pem") @@ -88,7 +88,7 @@ public void extendEnvironment(Environment.Builder builder) .withEnv("TTL_ACCESS_TOKEN", TTL_ACCESS_TOKEN_IN_SECONDS + "s") .withEnv("TTL_REFRESH_TOKEN", TTL_REFRESH_TOKEN_IN_SECONDS + "s") .withEnv("OAUTH2_ALLOWED_TOP_LEVEL_CLAIMS", "groups") - .withCommand("serve", "all", "--dangerous-force-http") + .withCommand("serve", "all") .withCopyFileToContainer(forHostPath(configDir.getPath("cert/hydra.pem")), "/tmp/certs/hydra.pem") .waitingFor(new WaitAllStrategy() .withStrategy(Wait.forLogMessage(".*Setting up http server on :4444.*", 1)) @@ -135,7 +135,7 @@ public DockerContainer createClient( { DockerContainer clientCreatingContainer = new DockerContainer(HYDRA_IMAGE, "hydra-client-preparation") .withCommand("clients", "create", - "--endpoint", "http://hydra:4445", + "--endpoint", "https://hydra:4445", "--skip-tls-verify", "--id", clientId, "--secret", clientSecret, diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Minio.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Minio.java index 941efd395cd7..fb2cb30800ab 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Minio.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/common/Minio.java @@ -20,6 +20,7 @@ import io.trino.tests.product.launcher.env.Environment; import io.trino.tests.product.launcher.testcontainers.PortBinder; import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; +import org.testcontainers.utility.DockerImageName; import java.time.Duration; @@ -39,7 +40,8 @@ public class Minio private static final String MINIO_ACCESS_KEY = "minio-access-key"; private static final String MINIO_SECRET_KEY = "minio-secret-key"; - private static final String MINIO_RELEASE = "RELEASE.2025-01-20T14-49-07Z"; + private static final String MINIO_RELEASE = DockerImageName.parse("cgr.dev/chainguard/minio@sha256:66bd82c8fe5e75868ae7d0b2e102d9a0dcf971b270a41bd060a9e6a643476ff8") + .asCanonicalNameString(); private static final int MINIO_PORT = 9080; // minio uses 9000 by default, which conflicts with hadoop private static final int MINIO_CONSOLE_PORT = 9001; @@ -69,13 +71,14 @@ public void extendEnvironment(Environment.Builder builder) private DockerContainer createMinioContainer() { - DockerContainer container = new DockerContainer("minio/minio:" + MINIO_RELEASE, MINIO_CONTAINER_NAME) + DockerContainer container = new DockerContainer(MINIO_RELEASE, MINIO_CONTAINER_NAME) .withEnv(ImmutableMap.builder() .put("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY) .put("MINIO_SECRET_KEY", MINIO_SECRET_KEY) .buildOrThrow()) .withCommand("server", "--address", format("0.0.0.0:%d", MINIO_PORT), "--console-address", format("0.0.0.0:%d", MINIO_CONSOLE_PORT), "/data") .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .withCreateContainerCmdModifier(cmd -> cmd.withUser("root")) // Required to create buckets externally .waitingFor(forSelectedPorts(MINIO_PORT)) .withStartupTimeout(Duration.ofMinutes(1)); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeHive4.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeHive4.java new file mode 100644 index 000000000000..8c8e0423334a --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeHive4.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.env.Environment; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.Hive4WithMinio; +import io.trino.tests.product.launcher.env.common.StandardMultinode; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public class EnvMultinodeHive4 + extends EnvironmentProvider +{ + private final DockerFiles.ResourceProvider configDir; + + @Inject + public EnvMultinodeHive4(StandardMultinode standardMultinode, DockerFiles dockerFiles, Hive4WithMinio hive4WithMinio) + { + super(ImmutableList.of(standardMultinode, hive4WithMinio)); + this.configDir = requireNonNull(dockerFiles, "dockerFiles is null").getDockerFilesHostDirectory("conf/environment/multinode-hive4"); + } + + @Override + public void extendEnvironment(Environment.Builder builder) + { + builder.addConnector("hive", forHostPath(configDir.getPath("trino/catalog/hive.properties"))); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeLoki.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeLoki.java new file mode 100644 index 000000000000..88f219767b6c --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeLoki.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.environment; + +import com.google.inject.Inject; +import io.trino.tests.product.launcher.docker.DockerFiles; +import io.trino.tests.product.launcher.docker.DockerFiles.ResourceProvider; +import io.trino.tests.product.launcher.env.DockerContainer; +import io.trino.tests.product.launcher.env.Environment.Builder; +import io.trino.tests.product.launcher.env.EnvironmentProvider; +import io.trino.tests.product.launcher.env.common.StandardMultinode; +import io.trino.tests.product.launcher.env.common.TestsEnvironment; +import io.trino.tests.product.launcher.testcontainers.PortBinder; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.time.Duration; + +import static java.util.Objects.requireNonNull; +import static org.testcontainers.utility.MountableFile.forHostPath; + +@TestsEnvironment +public class EnvMultinodeLoki + extends EnvironmentProvider +{ + public static final int LOKI_PORT = 3100; + + private final ResourceProvider configDir; + private final PortBinder portBinder; + + @Inject + public EnvMultinodeLoki(StandardMultinode standardMultinode, DockerFiles dockerFiles, PortBinder portBinder) + { + super(standardMultinode); + this.configDir = dockerFiles.getDockerFilesHostDirectory("conf/environment/multinode-loki/"); + this.portBinder = requireNonNull(portBinder, "portBinder is null"); + } + + @Override + public void extendEnvironment(Builder builder) + { + builder.addConnector("loki", forHostPath(configDir.getPath("loki.properties"))); + builder.addContainer(createLoki()); + } + + private DockerContainer createLoki() + { + DockerContainer container = new DockerContainer("grafana/loki:3.2.0", "loki") + .withExposedPorts(LOKI_PORT) + .waitingFor(Wait.forHttp("/ready").forResponsePredicate(response -> response.contains("ready"))) + .withStartupTimeout(Duration.ofMinutes(6)); + + portBinder.exposePort(container, LOKI_PORT); + + return container; + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks113.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks113.java deleted file mode 100644 index 54ea27443fbe..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks113.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.env.environment; - -import com.google.inject.Inject; -import io.trino.tests.product.launcher.docker.DockerFiles; -import io.trino.tests.product.launcher.env.common.Standard; -import io.trino.tests.product.launcher.env.common.TestsEnvironment; - -import static io.trino.testing.SystemEnvironmentUtils.requireEnv; - -@TestsEnvironment -public class EnvSinglenodeDeltaLakeDatabricks113 - extends AbstractSinglenodeDeltaLakeDatabricks -{ - @Inject - public EnvSinglenodeDeltaLakeDatabricks113(Standard standard, DockerFiles dockerFiles) - { - super(standard, dockerFiles); - } - - @Override - String databricksTestJdbcUrl() - { - return requireEnv("DATABRICKS_113_JDBC_URL"); - } -} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java index b1da087a8653..823d48730bb2 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks133.java @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks133(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireEnv("DATABRICKS_133_JDBC_URL"); + return requireEnv("DATABRICKS_133_JDBC_URL") + ";EnableArrow=0"; } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java index bec3c502b8b6..5ad2157f1b92 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks143.java @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks143(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireEnv("DATABRICKS_143_JDBC_URL"); + return requireEnv("DATABRICKS_143_JDBC_URL") + ";EnableArrow=0"; } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java index 37bb6d41eeb7..edfa7db9f382 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeDeltaLakeDatabricks154.java @@ -33,6 +33,6 @@ public EnvSinglenodeDeltaLakeDatabricks154(Standard standard, DockerFiles docker @Override String databricksTestJdbcUrl() { - return requireEnv("DATABRICKS_154_JDBC_URL"); + return requireEnv("DATABRICKS_154_JDBC_URL") + ";EnableArrow=0"; } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java index 5776cb0ef414..3fd8d6b89766 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java @@ -43,7 +43,7 @@ public class EnvSinglenodeSparkIcebergNessie private static final int SPARK_THRIFT_PORT = 10213; private static final int NESSIE_PORT = 19120; - private static final String NESSIE_VERSION = "0.105.3"; + private static final String NESSIE_VERSION = "0.105.6"; private static final String SPARK = "spark"; private final DockerFiles dockerFiles; diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/SpoolingMinio.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/SpoolingMinio.java index 07669e38a625..c15bcf5d0f56 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/SpoolingMinio.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/SpoolingMinio.java @@ -21,6 +21,7 @@ import io.trino.tests.product.launcher.env.common.EnvironmentExtender; import io.trino.tests.product.launcher.testcontainers.PortBinder; import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; +import org.testcontainers.utility.DockerImageName; import java.io.IOException; import java.io.UncheckedIOException; @@ -47,7 +48,8 @@ public class SpoolingMinio private static final String MINIO_SPOOLING_CONTAINER_NAME = "spooling-minio"; private static final String MINIO_ACCESS_KEY = "minio-access-key"; private static final String MINIO_SECRET_KEY = "minio-secret-key"; - private static final String MINIO_RELEASE = "RELEASE.2025-01-20T14-49-07Z"; + private static final String MINIO_RELEASE = DockerImageName.parse("cgr.dev/chainguard/minio@sha256:66bd82c8fe5e75868ae7d0b2e102d9a0dcf971b270a41bd060a9e6a643476ff8") + .asCanonicalNameString(); private static final int MINIO_PORT = 9080; // minio uses 9000 by default, which conflicts with hadoop private static final int MINIO_CONSOLE_PORT = 9001; @@ -95,7 +97,7 @@ private DockerContainer createSpoolingMinioContainer() throw new UncheckedIOException(e); } - DockerContainer container = new DockerContainer("minio/minio:" + MINIO_RELEASE, MINIO_SPOOLING_CONTAINER_NAME) + DockerContainer container = new DockerContainer(MINIO_RELEASE, MINIO_SPOOLING_CONTAINER_NAME) .withEnv(ImmutableMap.builder() .put("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY) .put("MINIO_SECRET_KEY", MINIO_SECRET_KEY) @@ -103,6 +105,7 @@ private DockerContainer createSpoolingMinioContainer() .withCopyFileToContainer(forHostPath(minioBucketDirectory), "/data/" + MINIO_SPOOLING_BUCKET) .withCommand("server", "--address", format("0.0.0.0:%d", MINIO_PORT), "--console-address", format("0.0.0.0:%d", MINIO_CONSOLE_PORT), "/data") .withStartupCheckStrategy(new IsRunningStartupCheckStrategy()) + .withCreateContainerCmdModifier(cmd -> cmd.withUser("root")) // Required to create buckets externally .waitingFor(forSelectedPorts(MINIO_PORT)) .withStartupTimeout(Duration.ofMinutes(1)); diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/BuiltInJdkProvider.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/BuiltInJdkProvider.java deleted file mode 100644 index dc5b84262a38..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/BuiltInJdkProvider.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.env.jdk; - -import io.airlift.log.Logger; -import io.trino.tests.product.launcher.env.DockerContainer; - -public class BuiltInJdkProvider - implements JdkProvider -{ - private final Logger log = Logger.get(getClass()); - - public static final String BUILT_IN_NAME = "builtin"; - - @Override - public DockerContainer applyTo(DockerContainer container) - { - log.info("Setting JAVA_HOME to: %s for container: %s", getJavaHome(), container.getLogicalName()); - return container.withEnv("JAVA_HOME", getJavaHome()); - } - - @Override - public String getJavaHome() - { - // This is provided by docker image - return "/usr/lib/jvm/zulu-17"; - } - - @Override - public String getDescription() - { - return "JDK provider by base image"; - } -} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/DistributionDownloadingJdkProvider.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/DistributionDownloadingJdkProvider.java deleted file mode 100644 index 58159907c223..000000000000 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/DistributionDownloadingJdkProvider.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.tests.product.launcher.env.jdk; - -import io.trino.testing.containers.TestContainers; -import jakarta.annotation.Nullable; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Properties; - -import static java.util.Objects.requireNonNull; - -public final class DistributionDownloadingJdkProvider - extends TarDownloadingJdkProvider -{ - private final String distributionName; - private final Path distributionPath; - - public DistributionDownloadingJdkProvider(String distributionsPath, String distributionName, @Nullable Path downloadPath) - { - super(downloadPath); - this.distributionName = requireNonNull(distributionName, "distributionName is null"); - this.distributionPath = Paths.get(distributionsPath).resolve(distributionName); - } - - @Override - public String getDescription() - { - return distributionName; - } - - @Override - protected String getName() - { - return distributionName.replace("/", "_"); - } - - @Override - protected String getDownloadUri(TestContainers.DockerArchitecture architecture) - { - Properties properties = new Properties(); - try (InputStream inputStream = Files.newInputStream(getDistributionPath(distributionPath, architecture))) { - properties.load(inputStream); - return requireNonNull(properties.getProperty("distributionUrl"), "distributionUrl is null").trim(); - } - catch (IOException e) { - throw new IllegalArgumentException("Architecture %s not found in distribution %s".formatted(architecture, distributionPath)); - } - } - - private static Path getDistributionPath(Path distributionPath, TestContainers.DockerArchitecture architecture) - { - return switch (architecture) { - case AMD64 -> distributionPath.resolve("amd64"); - case ARM64 -> distributionPath.resolve("arm64"); - case PPC64 -> distributionPath.resolve("ppc64le"); - }; - } -} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/TemurinJdkProvider.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/TemurinJdkProvider.java new file mode 100644 index 000000000000..09c634ab832b --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/jdk/TemurinJdkProvider.java @@ -0,0 +1,62 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.env.jdk; + +import io.trino.testing.containers.TestContainers; +import jakarta.annotation.Nullable; + +import java.nio.file.Path; + +import static java.util.Objects.requireNonNull; + +public final class TemurinJdkProvider + extends TarDownloadingJdkProvider +{ + private static final String TEMURIN_DOWNLOAD_URL = "https://api.adoptium.net/v3/binary/version/%s/linux/%s/jdk/hotspot/normal/eclipse?project=jdk"; + + private final String releaseName; + + public TemurinJdkProvider(String releaseName, @Nullable Path downloadPath) + { + super(downloadPath); + this.releaseName = requireNonNull(releaseName, "releaseName is null"); + } + + @Override + public String getDescription() + { + return releaseName; + } + + @Override + protected String getName() + { + return "Temurin " + releaseName.replace("/", "_"); + } + + @Override + protected String getDownloadUri(TestContainers.DockerArchitecture architecture) + { + return TEMURIN_DOWNLOAD_URL.formatted(releaseName, getTemurinArchitectureName(architecture)); + } + + private static String getTemurinArchitectureName(TestContainers.DockerArchitecture architecture) + { + return switch (architecture) { + case AMD64 -> "x64"; + case ARM64 -> "aarch64"; + case PPC64 -> "ppc64le"; + }; + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks122.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks122.java index ce48cd75fbf2..9eccd7c9089b 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks122.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks122.java @@ -22,7 +22,7 @@ import java.util.List; import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; public class SuiteDeltaLakeDatabricks122 @@ -33,7 +33,7 @@ public List getTestRuns(EnvironmentConfig config) { return ImmutableList.of( testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks122.class) - .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS_122) + .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS) .withExcludedTests(getExcludedTests()) .build()); } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks113.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteHive4.java similarity index 62% rename from testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks113.java rename to testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteHive4.java index ae9f51cfb3ec..559e9d90bcc1 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteDeltaLakeDatabricks113.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteHive4.java @@ -15,28 +15,25 @@ import com.google.common.collect.ImmutableList; import io.trino.tests.product.launcher.env.EnvironmentConfig; -import io.trino.tests.product.launcher.env.environment.EnvSinglenodeDeltaLakeDatabricks113; -import io.trino.tests.product.launcher.suite.SuiteDeltaLakeDatabricks; +import io.trino.tests.product.launcher.env.environment.EnvMultinodeHive4; +import io.trino.tests.product.launcher.suite.Suite; import io.trino.tests.product.launcher.suite.SuiteTestRun; import java.util.List; import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; +import static io.trino.tests.product.TestGroups.HIVE4; import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; -public class SuiteDeltaLakeDatabricks113 - extends SuiteDeltaLakeDatabricks +public class SuiteHive4 + extends Suite { @Override public List getTestRuns(EnvironmentConfig config) { return ImmutableList.of( - testOnEnvironment(EnvSinglenodeDeltaLakeDatabricks113.class) - .withGroups(CONFIGURED_FEATURES, DELTA_LAKE_DATABRICKS) - .withExcludedGroups(DELTA_LAKE_EXCLUDE_113) - .withExcludedTests(getExcludedTests()) + testOnEnvironment(EnvMultinodeHive4.class) + .withGroups(HIVE4, CONFIGURED_FEATURES) .build()); } } diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteLoki.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteLoki.java new file mode 100644 index 000000000000..2718d340e63d --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/suite/suites/SuiteLoki.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.launcher.suite.suites; + +import com.google.common.collect.ImmutableList; +import io.trino.tests.product.launcher.env.EnvironmentConfig; +import io.trino.tests.product.launcher.env.environment.EnvMultinodeLoki; +import io.trino.tests.product.launcher.suite.Suite; +import io.trino.tests.product.launcher.suite.SuiteTestRun; + +import java.util.List; + +import static io.trino.tests.product.launcher.suite.SuiteTestRun.testOnEnvironment; + +public class SuiteLoki + extends Suite +{ + @Override + public List getTestRuns(EnvironmentConfig config) + { + return ImmutableList.of( + testOnEnvironment(EnvMultinodeLoki.class) + .withGroups("configured_features", "loki") + .build()); + } +} diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/testcontainers/ExistingNetwork.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/testcontainers/ExistingNetwork.java index cd58d792c8bf..4a70bbc554dc 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/testcontainers/ExistingNetwork.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/testcontainers/ExistingNetwork.java @@ -13,8 +13,6 @@ */ package io.trino.tests.product.launcher.testcontainers; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; import org.testcontainers.containers.Network; import static java.util.Objects.requireNonNull; @@ -37,11 +35,4 @@ public String getId() @Override public void close() {} - - @Override - public Statement apply(Statement statement, Description description) - { - // junit4 integration - throw new UnsupportedOperationException(); - } } diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/hive-site.xml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/hive-site.xml new file mode 100644 index 000000000000..ca322685d25c --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/hive-site.xml @@ -0,0 +1,80 @@ + + + + hive.server2.enable.doAs + false + + + hive.tez.exec.inplace.progress + false + + + hive.exec.scratchdir + /opt/hive/scratch_dir + + + hive.user.install.directory + /opt/hive/install_dir + + + tez.runtime.optimize.local.fetch + true + + + hive.exec.submit.local.task.via.child + false + + + mapreduce.framework.name + local + + + tez.local.mode + true + + + hive.execution.engine + tez + + + hive.metastore.warehouse.dir + s3a://test-bucket/ + + + metastore.metastore.event.db.notification.api.auth + false + + + + + hive.users.in.admin.role + hive + + + + + fs.s3a.access.key + minio-access-key + + + fs.s3a.secret.key + minio-secret-key + + + fs.s3a.endpoint + http://minio:9080 + + + fs.s3a.path.style.access + true + + + fs.s3.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3n.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/tempto-configuration.yaml new file mode 100644 index 000000000000..c16b1d16a079 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hive4-with-minio/tempto-configuration.yaml @@ -0,0 +1,18 @@ +databases: + hive: + host: hiveserver2 + jdbc_driver_class: org.apache.hive.jdbc.HiveDriver + jdbc_url: jdbc:hive2://${databases.hive.host}:10000 + jdbc_user: hive + jdbc_password: na + jdbc_pooling: false + schema: default + prepare_statement: + - USE ${databases.hive.schema} + # Hive 4 gathers stats by default. For test purposes we need to disable this behavior. + - SET hive.stats.column.autogather=false + table_manager_type: hive + warehouse_directory_path: s3a://${S3_BUCKET}/ + metastore: + host: metastore + port: 9083 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hydra-identity-provider/login_and_consent_server.py b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hydra-identity-provider/login_and_consent_server.py index bcf9a37ed7bb..d1a08f456170 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hydra-identity-provider/login_and_consent_server.py +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/common/hydra-identity-provider/login_and_consent_server.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse, parse_qs from urllib.request import Request, urlopen -HYDRA_ADMIN_URL = os.getenv("HYDRA_ADMIN_URL", "http://hydra:4445") +HYDRA_ADMIN_URL = os.getenv("HYDRA_ADMIN_URL", "https://hydra:4445") PORT = os.getenv("PORT", 3000) SSL_CONTEXT = ssl.create_default_context() SSL_CONTEXT.check_hostname = False diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/jvm.config b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/jvm.config index 92425b256347..90d2ff6a3a5f 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/jvm.config +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-all/jvm.config @@ -1,6 +1,5 @@ -server --add-opens=java.base/java.nio=ALL-UNNAMED ---sun-misc-unsafe-memory-access=allow -Xmx2G -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-hive4/trino/catalog/hive.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-hive4/trino/catalog/hive.properties new file mode 100644 index 000000000000..9f1d8a437c84 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-hive4/trino/catalog/hive.properties @@ -0,0 +1,13 @@ +connector.name=hive +hive.metastore.uri=thrift://metastore:9083 +hive.non-managed-table-writes-enabled=true +fs.native-s3.enabled=true +fs.hadoop.enabled=false +s3.region=us-east-1 +s3.aws-access-key=minio-access-key +s3.aws-secret-key=minio-secret-key +s3.endpoint=http://minio:9080/ +s3.path-style-access=true + +hive.parquet.time-zone=UTC +hive.rcfile.time-zone=UTC diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-loki/loki.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-loki/loki.properties new file mode 100644 index 000000000000..7f568fcd7489 --- /dev/null +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-loki/loki.properties @@ -0,0 +1,2 @@ +connector.name=loki +loki.uri=http://loki:3100 diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-snowflake/jvm.config b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-snowflake/jvm.config index da5de4ee9d60..491f05d67917 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-snowflake/jvm.config +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/multinode-snowflake/jvm.config @@ -1,6 +1,5 @@ -server --add-opens=java.base/java.nio=ALL-UNNAMED ---sun-misc-unsafe-memory-access=allow -Xmx2G -XX:G1HeapRegionSize=32M -XX:+ExplicitGCInvokesConcurrent diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml index 95f53a37bf93..77826c140919 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-delta-lake-databricks/tempto-configuration.yaml @@ -8,7 +8,7 @@ databases: prepare_statement: - USE ${databases.delta.schema} table_manager_type: jdbc - jdbc_url: ${DATABRICKS_JDBC_URL};EnableArrow=0;SocketTimeout=120 + jdbc_url: ${DATABRICKS_JDBC_URL};SocketTimeout=120 jdbc_user: ${DATABRICKS_LOGIN} jdbc_password: ${DATABRICKS_TOKEN} diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-http-proxy/trino/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-http-proxy/trino/config.properties index 6921e4429f48..0167599bb3e5 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-http-proxy/trino/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-http-proxy/trino/config.properties @@ -13,10 +13,10 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ -http-server.authentication.oauth2.auth-url=http://hydra:4444/oauth2/auth -http-server.authentication.oauth2.token-url=http://hydra:4444/oauth2/token -http-server.authentication.oauth2.jwks-url=http://hydra:4444/.well-known/jwks.json +http-server.authentication.oauth2.issuer=https://hydra:4444/ +http-server.authentication.oauth2.auth-url=https://hydra:4444/oauth2/auth +http-server.authentication.oauth2.token-url=https://hydra:4444/oauth2/token +http-server.authentication.oauth2.jwks-url=https://hydra:4444/.well-known/jwks.json http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret http-server.authentication.oauth2.user-mapping.pattern=(.*)(@.*)? diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-https-proxy/trino/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-https-proxy/trino/config.properties index 22fd8ca53047..e5ae90b9dcf3 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-https-proxy/trino/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-authenticated-https-proxy/trino/config.properties @@ -13,10 +13,10 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ -http-server.authentication.oauth2.auth-url=http://hydra:4444/oauth2/auth -http-server.authentication.oauth2.token-url=http://hydra:4444/oauth2/token -http-server.authentication.oauth2.jwks-url=http://hydra:4444/.well-known/jwks.json +http-server.authentication.oauth2.issuer=https://hydra:4444/ +http-server.authentication.oauth2.auth-url=https://hydra:4444/oauth2/auth +http-server.authentication.oauth2.token-url=https://hydra:4444/oauth2/token +http-server.authentication.oauth2.jwks-url=https://hydra:4444/.well-known/jwks.json http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret http-server.authentication.oauth2.user-mapping.pattern=(.*)(@.*)? diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties index 0b78bb5829ff..2bd29955d35e 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-http-proxy/config.properties @@ -13,10 +13,10 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ -http-server.authentication.oauth2.auth-url=http://hydra:4444/oauth2/auth -http-server.authentication.oauth2.token-url=http://hydra:4444/oauth2/token -http-server.authentication.oauth2.jwks-url=http://hydra:4444/.well-known/jwks.json +http-server.authentication.oauth2.issuer=https://hydra:4444/ +http-server.authentication.oauth2.auth-url=https://hydra:4444/oauth2/auth +http-server.authentication.oauth2.token-url=https://hydra:4444/oauth2/token +http-server.authentication.oauth2.jwks-url=https://hydra:4444/.well-known/jwks.json http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret http-server.authentication.oauth2.user-mapping.pattern=(.*)(@.*)? diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties index 75dc8a0d4762..887dbeb60b82 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-https-proxy/config.properties @@ -13,10 +13,10 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ -http-server.authentication.oauth2.auth-url=http://hydra:4444/oauth2/auth -http-server.authentication.oauth2.token-url=http://hydra:4444/oauth2/token -http-server.authentication.oauth2.jwks-url=http://hydra:4444/.well-known/jwks.json +http-server.authentication.oauth2.issuer=https://hydra:4444/ +http-server.authentication.oauth2.auth-url=https://hydra:4444/oauth2/auth +http-server.authentication.oauth2.token-url=https://hydra:4444/oauth2/token +http-server.authentication.oauth2.jwks-url=https://hydra:4444/.well-known/jwks.json http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret http-server.authentication.oauth2.user-mapping.pattern=(.*)(@.*)? diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-refresh/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-refresh/config.properties index c1bafd54df77..c8a5e2776af2 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-refresh/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2-refresh/config.properties @@ -13,11 +13,11 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ +http-server.authentication.oauth2.issuer=https://hydra:4444/ http-server.authentication.oauth2.scopes=openid,offline -http-server.authentication.oauth2.auth-url=http://hydra:4444/oauth2/auth -http-server.authentication.oauth2.token-url=http://hydra:4444/oauth2/token -http-server.authentication.oauth2.jwks-url=http://hydra:4444/.well-known/jwks.json +http-server.authentication.oauth2.auth-url=https://hydra:4444/oauth2/auth +http-server.authentication.oauth2.token-url=https://hydra:4444/oauth2/token +http-server.authentication.oauth2.jwks-url=https://hydra:4444/.well-known/jwks.json http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret http-server.authentication.oauth2.user-mapping.pattern=(.*)(@.*)? diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2/config.properties index 352359c6e46d..448bb9f1f8fa 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oauth2/config.properties @@ -13,10 +13,10 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ -http-server.authentication.oauth2.auth-url=http://hydra:4444/oauth2/auth -http-server.authentication.oauth2.token-url=http://hydra:4444/oauth2/token -http-server.authentication.oauth2.jwks-url=http://hydra:4444/.well-known/jwks.json +http-server.authentication.oauth2.issuer=https://hydra:4444/ +http-server.authentication.oauth2.auth-url=https://hydra:4444/oauth2/auth +http-server.authentication.oauth2.token-url=https://hydra:4444/oauth2/token +http-server.authentication.oauth2.jwks-url=https://hydra:4444/.well-known/jwks.json http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret http-server.authentication.oauth2.user-mapping.pattern=(.*)(@.*)? diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc-refresh/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc-refresh/config.properties index b8a83115131f..f670fac6b1ab 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc-refresh/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc-refresh/config.properties @@ -13,7 +13,7 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ +http-server.authentication.oauth2.issuer=https://hydra:4444/ http-server.authentication.oauth2.scopes=openid,offline http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc/config.properties b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc/config.properties index c58d901e1555..f0714f797938 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc/config.properties +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/environment/singlenode-oidc/config.properties @@ -13,7 +13,7 @@ http-server.authentication.type=oauth2 http-server.https.port=7778 http-server.https.enabled=true http-server.https.keystore.path=/docker/trino-product-tests/conf/trino/etc/trino.pem -http-server.authentication.oauth2.issuer=http://hydra:4444/ +http-server.authentication.oauth2.issuer=https://hydra:4444/ http-server.authentication.oauth2.client-id=trinodb_client_id http-server.authentication.oauth2.client-secret=trinodb_client_secret http-server.authentication.oauth2.user-mapping.pattern=(.*)(@.*)? diff --git a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/trino/etc/jvm.config b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/trino/etc/jvm.config index ef629960d561..78ac5560a252 100644 --- a/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/trino/etc/jvm.config +++ b/testing/trino-product-tests-launcher/src/main/resources/docker/trino-product-tests/conf/trino/etc/jvm.config @@ -14,6 +14,3 @@ -XX:ErrorFile=/docker/logs/product-tests-presto-jvm-error-file.log # Allow loading dynamic agent used by JOL -XX:+EnableDynamicAgentLoading --XX:+UnlockDiagnosticVMOptions ---sun-misc-unsafe-memory-access=allow --XX:+IgnoreUnrecognizedVMOptions diff --git a/testing/trino-product-tests-launcher/src/test/java/io/trino/tests/product/launcher/env/TestConfigurations.java b/testing/trino-product-tests-launcher/src/test/java/io/trino/tests/product/launcher/env/TestConfigurations.java index 48541cbe9db5..19f3344fa5b6 100644 --- a/testing/trino-product-tests-launcher/src/test/java/io/trino/tests/product/launcher/env/TestConfigurations.java +++ b/testing/trino-product-tests-launcher/src/test/java/io/trino/tests/product/launcher/env/TestConfigurations.java @@ -14,7 +14,7 @@ package io.trino.tests.product.launcher.env; import io.trino.tests.product.launcher.env.environment.EnvMultinodeSqlserver; -import io.trino.tests.product.launcher.env.jdk.BuiltInJdkProvider; +import io.trino.tests.product.launcher.env.jdk.TemurinJdkProvider; import io.trino.tests.product.launcher.suite.suites.Suite1; import io.trino.tests.product.launcher.suite.suites.Suite6NonGeneric; import io.trino.tests.product.launcher.suite.suites.SuiteTpcds; @@ -61,7 +61,7 @@ public void testSuiteName() @Test public void testJdkProviderName() { - assertThat(nameForJdkProviderName(BuiltInJdkProvider.class)).isEqualTo("builtin"); + assertThat(nameForJdkProviderName(TemurinJdkProvider.class)).isEqualTo("temurin"); assertThat(canonicalJdkProviderName("BuiltIN")).isEqualTo("builtin"); assertThat(canonicalJdkProviderName("built-IN")).isEqualTo("builtin"); } diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index 02382ba297da..5799c1a69a14 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -98,6 +98,11 @@ kafka-schema-registry-client + + io.github.jeschkies + loki-client + + io.minio minio diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestConfiguredFeatures.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestConfiguredFeatures.java index 7aa4eccf2a2e..62d41910cd2b 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestConfiguredFeatures.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/TestConfiguredFeatures.java @@ -16,14 +16,16 @@ import com.google.inject.Inject; import com.google.inject.name.Named; import io.trino.tempto.ProductTest; +import io.trino.tempto.assertions.QueryAssert; import org.testng.SkipException; import org.testng.annotations.Test; import java.util.List; +import java.util.stream.Collectors; -import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.tests.product.TestGroups.CONFIGURED_FEATURES; import static io.trino.tests.product.utils.QueryExecutors.onTrino; +import static java.sql.JDBCType.VARCHAR; import static org.assertj.core.api.Assertions.assertThat; /** @@ -47,16 +49,10 @@ public void selectConfiguredConnectors() throw new SkipException("Skip checking configured connectors since none were set in Tempto configuration"); } String sql = "SELECT DISTINCT connector_name FROM system.metadata.catalogs"; - List loadedCatalogs = onTrino().executeQuery(sql).column(1).stream() - .map(Object::toString) - .collect(toImmutableList()); - // TODO https://github.com/trinodb/trino/issues/26500 - // Loki connector is not loading properly. Once this is fixed, test will fail. - List filteredCatalogs = configuredConnectors.stream() - .filter(connector -> !connector.equals("loki")) - .collect(toImmutableList()); - - assertThat(filteredCatalogs) - .containsExactlyInAnyOrder(loadedCatalogs.toArray(new String[0])); + assertThat(onTrino().executeQuery(sql)) + .hasColumns(VARCHAR) + .containsOnly(configuredConnectors.stream() + .map(QueryAssert.Row::row) + .collect(Collectors.toList())); } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java index e179bdefc3a9..a186a0338374 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeAlterTableCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_143_RUNTIME_VERSION; @@ -242,7 +241,7 @@ public void testTrinoPreservesReaderAndWriterVersions() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoPreservesTableFeature() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java index ad7923a00c1d..23d68e51b0c8 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeChangeDataFeedCompatibility.java @@ -32,11 +32,9 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client; @@ -108,7 +106,7 @@ public void testUpdateTableWithCdf(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdateTableWithChangeDataFeedWriterFeature() { @@ -541,7 +539,7 @@ public void testMergeDeleteIntoTableWithCdfEnabled(String columnMappingMode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testMergeMixedDeleteAndUpdateIntoTableWithCdfEnabled() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java index 5500030ffc20..310ba55ebcc1 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCheckpointsCompatibility.java @@ -21,7 +21,6 @@ import io.trino.tempto.assertions.QueryAssert.Row; import io.trino.tempto.query.QueryResult; import io.trino.testng.services.Flaky; -import io.trino.tests.product.deltalake.util.DatabricksVersion; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import software.amazon.awssdk.services.s3.S3Client; @@ -29,7 +28,6 @@ import java.math.BigDecimal; import java.util.List; -import java.util.Optional; import java.util.function.Consumer; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -37,7 +35,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154; @@ -46,11 +43,9 @@ import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client; import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertLastEntryIsCheckpointed; import static io.trino.tests.product.deltalake.TransactionLogAssertions.assertTransactionLogVersion; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry; -import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getDatabricksRuntimeVersion; import static io.trino.tests.product.utils.QueryExecutors.onDelta; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; @@ -64,13 +59,11 @@ public class TestDeltaLakeCheckpointsCompatibility private String s3ServerType; private S3Client s3; - private Optional databricksRuntimeVersion; @BeforeMethodWithContext public void setup() { s3 = createS3Client(s3ServerType); - databricksRuntimeVersion = getDatabricksRuntimeVersion(); } @AfterMethodWithContext @@ -283,7 +276,7 @@ private void trinoUsesCheckpointInterval(String deltaTableProperties) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDatabricksUsesCheckpointInterval() { @@ -375,7 +368,7 @@ private void testCheckpointMinMaxStatisticsForRowType(Consumer sqlExecut // Assert min/max queries can be computed from just metadata String explainSelectMax = getOnlyElement(onDelta().executeQuery("EXPLAIN SELECT max(root.entry_one) FROM default." + tableName).column(1)); - String column = databricksRuntimeVersion.orElseThrow().isAtLeast(DATABRICKS_113_RUNTIME_VERSION) ? "root.entry_one" : "root.entry_one AS `entry_one`"; + String column = "root.entry_one"; assertThat(explainSelectMax).matches("== Physical Plan ==\\s*LocalTableScan \\[max\\(" + column + "\\).*]\\s*"); // check both engines can read both tables @@ -441,7 +434,7 @@ private void testCheckpointNullStatisticsForRowType(Consumer sqlExecutor // Assert counting non null entries can be computed from just metadata String explainCountNotNull = getOnlyElement(onDelta().executeQuery("EXPLAIN SELECT count(root.entry_two) FROM default." + tableName).column(1)); - String column = databricksRuntimeVersion.orElseThrow().isAtLeast(DATABRICKS_113_RUNTIME_VERSION) ? "root.entry_two" : "root.entry_two AS `entry_two`"; + String column = "root.entry_two"; assertThat(explainCountNotNull).matches("== Physical Plan ==\\s*LocalTableScan \\[count\\(" + column + "\\).*]\\s*"); // check both engines can read both tables diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java index ae75b95bfa62..2eb76bdc4053 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeColumnMappingMode.java @@ -27,7 +27,6 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154; @@ -54,7 +53,7 @@ public class TestDeltaLakeColumnMappingMode extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testColumnMappingModeNone() { @@ -227,7 +226,7 @@ private void testColumnMappingModeReaderAndWriterVersion(Consumer create onTrino().executeQuery("DROP TABLE delta.default." + tableName); } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoColumnMappingMode(String mode) { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java index 50422546f0d7..70daa3f31a01 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeCreateTableAsSelectCompatibility.java @@ -25,7 +25,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154; @@ -43,7 +42,7 @@ public class TestDeltaLakeCreateTableAsSelectCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoTypesWithDatabricks() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java index 07b8f3abdd89..2f549a5b2567 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDatabricksCreateTableCompatibility.java @@ -14,10 +14,8 @@ package io.trino.tests.product.deltalake; import com.google.common.collect.ImmutableList; -import io.trino.tempto.BeforeMethodWithContext; import io.trino.tempto.assertions.QueryAssert; import io.trino.testng.services.Flaky; -import io.trino.tests.product.deltalake.util.DatabricksVersion; import org.testng.annotations.Test; import java.util.List; @@ -26,13 +24,11 @@ import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnDelta; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getColumnCommentOnTrino; -import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getDatabricksRuntimeVersion; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getTableCommentOnDelta; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getTableCommentOnTrino; import static io.trino.tests.product.utils.QueryExecutors.onDelta; @@ -44,14 +40,6 @@ public class TestDeltaLakeDatabricksCreateTableCompatibility extends BaseTestDeltaLakeS3Storage { - private DatabricksVersion databricksRuntimeVersion; - - @BeforeMethodWithContext - public void setup() - { - databricksRuntimeVersion = getDatabricksRuntimeVersion().orElseThrow(); - } - @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDatabricksCanReadInitialCreateTable() @@ -67,23 +55,12 @@ public void testDatabricksCanReadInitialCreateTable() try { assertThat(onDelta().executeQuery("SHOW TABLES FROM default LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM default." + tableName)).contains(row(0)); - String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { - showCreateTable = format( - "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\nLOCATION 's3://%s/%s'\n%s", - tableName, - bucketName, - tableDirectory, - getDatabricksDefaultTableProperties()); - } - else { - showCreateTable = format( - "CREATE TABLE `default`.`%s` (\n `integer` INT,\n `string` STRING,\n `timetz` TIMESTAMP)\nUSING DELTA\nLOCATION 's3://%s/%s'\n" + - "TBLPROPERTIES (\n 'delta.enableDeletionVectors' = 'false')\n", - tableName, - bucketName, - tableDirectory); - } + String showCreateTable = format( + "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\nLOCATION 's3://%s/%s'\n%s", + tableName, + bucketName, + tableDirectory, + getDatabricksDefaultTableProperties()); assertThat(onDelta().executeQuery("SHOW CREATE TABLE default." + tableName)) .containsExactlyInOrder(row(showCreateTable)); testInsert(tableName, ImmutableList.of()); @@ -110,25 +87,13 @@ public void testDatabricksCanReadInitialCreatePartitionedTable() try { assertThat(onDelta().executeQuery("SHOW TABLES LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM " + tableName)).contains(row(0)); - String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { - showCreateTable = format( + String showCreateTable = format( "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\n" + "PARTITIONED BY (string)\nLOCATION 's3://%s/%s'\n%s", tableName, bucketName, tableDirectory, getDatabricksDefaultTableProperties()); - } - else { - showCreateTable = format( - "CREATE TABLE `default`.`%s` (\n `integer` INT,\n `string` STRING,\n `timetz` TIMESTAMP)\nUSING DELTA\n" + - "PARTITIONED BY (string)\nLOCATION 's3://%s/%s'\n" + - "TBLPROPERTIES (\n 'delta.enableDeletionVectors' = 'false')\n", - tableName, - bucketName, - tableDirectory); - } assertThat(onDelta().executeQuery("SHOW CREATE TABLE " + tableName)).containsExactlyInOrder(row(showCreateTable)); testInsert(tableName, ImmutableList.of()); } @@ -153,23 +118,12 @@ public void testDatabricksCanReadInitialCreateTableAs() try { assertThat(onDelta().executeQuery("SHOW TABLES FROM default LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM default." + tableName)).contains(row(3)); - String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { - showCreateTable = format( + String showCreateTable = format( "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\nLOCATION 's3://%s/%s'\n%s", tableName, bucketName, tableDirectory, getDatabricksDefaultTableProperties()); - } - else { - showCreateTable = format( - "CREATE TABLE `default`.`%s` (\n `integer` INT,\n `string` STRING,\n `timetz` TIMESTAMP)\nUSING DELTA\nLOCATION 's3://%s/%s'\n" + - "TBLPROPERTIES (\n 'delta.enableDeletionVectors' = 'false')\n", - tableName, - bucketName, - tableDirectory); - } assertThat(onDelta().executeQuery("SHOW CREATE TABLE default." + tableName)).containsExactlyInOrder(row(showCreateTable)); testInsert( tableName, @@ -199,25 +153,13 @@ public void testDatabricksCanReadInitialCreatePartitionedTableAs() try { assertThat(onDelta().executeQuery("SHOW TABLES LIKE '" + tableName + "'")).contains(row("default", tableName, false)); assertThat(onDelta().executeQuery("SELECT count(*) FROM " + tableName)).contains(row(3)); - String showCreateTable; - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { - showCreateTable = format( + String showCreateTable = format( "CREATE TABLE spark_catalog.default.%s (\n integer INT,\n string STRING,\n timetz TIMESTAMP)\nUSING delta\n" + "PARTITIONED BY (string)\nLOCATION 's3://%s/%s'\n%s", tableName, bucketName, tableDirectory, getDatabricksDefaultTableProperties()); - } - else { - showCreateTable = format( - "CREATE TABLE `default`.`%s` (\n `integer` INT,\n `string` STRING,\n `timetz` TIMESTAMP)\nUSING DELTA\n" + - "PARTITIONED BY (string)\nLOCATION 's3://%s/%s'\n" + - "TBLPROPERTIES (\n 'delta.enableDeletionVectors' = 'false')\n", - tableName, - bucketName, - tableDirectory); - } assertThat(onDelta().executeQuery("SHOW CREATE TABLE " + tableName)).containsExactlyInOrder(row(showCreateTable)); testInsert( tableName, @@ -398,14 +340,11 @@ public void testCreateTableWithAllPartitionColumns() } } - private String getDatabricksDefaultTableProperties() + private static String getDatabricksDefaultTableProperties() { - if (databricksRuntimeVersion.isAtLeast(DATABRICKS_113_RUNTIME_VERSION)) { - return "TBLPROPERTIES (\n" + - " 'delta.enableDeletionVectors' = 'false',\n" + - " 'delta.minReaderVersion' = '1',\n" + - " 'delta.minWriterVersion' = '2')\n"; - } - throw new IllegalArgumentException("Unsupported databricks runtime version: " + databricksRuntimeVersion); + return "TBLPROPERTIES (\n" + + " 'delta.enableDeletionVectors' = 'false',\n" + + " 'delta.minReaderVersion' = '1',\n" + + " 'delta.minWriterVersion' = '2')\n"; } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java index 23ad38b572aa..b588c3b5c3f1 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeDeleteCompatibility.java @@ -31,7 +31,6 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_EXCLUDE_113; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; @@ -208,7 +207,7 @@ public void testTrinoDeletionVectors() } // Databricks 12.1 and OSS Delta 2.4.0 added support for deletion vectors - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingModeDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeletionVectors(String mode) { @@ -495,7 +494,7 @@ public void testDeletionVectorsAcrossAddFile(boolean partitioned) } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_EXCLUDE_113, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDeletionVectorsTruncateTable() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java index eb7d60a1d6af..436ad9336ad5 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeIdentityColumnCompatibility.java @@ -21,7 +21,7 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; @@ -37,7 +37,7 @@ public class TestDeltaLakeIdentityColumnCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_133, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumn() { @@ -77,7 +77,7 @@ public void testIdentityColumn() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS_133, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testRenameIdentityColumn(String mode) { @@ -114,7 +114,7 @@ public void testRenameIdentityColumn(String mode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") + @Test(groups = {DELTA_LAKE_DATABRICKS_133, PROFILE_SPECIFIC_TESTS}, dataProvider = "columnMappingDataProvider") @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testDropIdentityColumn(String mode) { @@ -152,7 +152,7 @@ public void testDropIdentityColumn(String mode) } } - @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_133, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testVacuumProcedureWithIdentityColumn() { @@ -183,7 +183,7 @@ public void testVacuumProcedureWithIdentityColumn() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_133, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testIdentityColumnCheckpointInterval() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java index 5eb306c37c0e..3afdaf184e00 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeInsertCompatibility.java @@ -14,15 +14,12 @@ package io.trino.tests.product.deltalake; import com.google.common.collect.ImmutableList; -import io.trino.tempto.BeforeMethodWithContext; import io.trino.tempto.assertions.QueryAssert.Row; import io.trino.testng.services.Flaky; -import io.trino.tests.product.deltalake.util.DatabricksVersion; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; import java.util.List; -import java.util.Optional; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -31,17 +28,14 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; -import static io.trino.tests.product.deltalake.util.DatabricksVersion.DATABRICKS_113_RUNTIME_VERSION; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.dropDeltaTableWithRetry; -import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.getDatabricksRuntimeVersion; import static io.trino.tests.product.utils.QueryExecutors.onDelta; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.util.Arrays.asList; @@ -50,15 +44,7 @@ public class TestDeltaLakeInsertCompatibility extends BaseTestDeltaLakeS3Storage { - private Optional databricksRuntimeVersion; - - @BeforeMethodWithContext - public void setup() - { - databricksRuntimeVersion = getDatabricksRuntimeVersion(); - } - - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testInsertCompatibility() { @@ -94,7 +80,7 @@ public void testInsertCompatibility() } } - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testPartitionedInsertCompatibility() { @@ -414,14 +400,8 @@ public void testCompression(String compressionCodec) assertThat(onTrino().executeQuery("SELECT * FROM " + trinoTableName)) .containsOnly(expected); - if ("ZSTD".equals(compressionCodec) && databricksRuntimeVersion.orElseThrow().isOlderThan(DATABRICKS_113_RUNTIME_VERSION)) { - assertQueryFailure(() -> onDelta().executeQuery("SELECT * FROM default." + tableName)) - .hasMessageContaining("java.lang.ClassNotFoundException: org.apache.hadoop.io.compress.ZStandardCodec"); - } - else { - assertThat(onDelta().executeQuery("SELECT * FROM default." + tableName)) - .containsOnly(expected); - } + assertThat(onDelta().executeQuery("SELECT * FROM default." + tableName)) + .containsOnly(expected); } } finally { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java index 6e92f6953bf4..8cf53808daad 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java @@ -34,6 +34,7 @@ public class TestDeltaLakeJmx public void testJmxTablesExposedByDeltaLakeConnectorBackedByGlueMetastore() { assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( + row("io.airlift.bootstrap:name=delta,type=lifecyclemanager"), row("io.trino.filesystem.s3:name=delta,type=s3filesystemstats"), row("io.trino.metastore.cache:name=delta,type=cachinghivemetastore"), row("io.trino.plugin.hive.metastore.glue:name=delta,type=gluehivemetastore"), @@ -47,6 +48,7 @@ public void testJmxTablesExposedByDeltaLakeConnectorBackedByGlueMetastore() public void testJmxTablesExposedByDeltaLakeConnectorBackedByThriftMetastore() { assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( + row("io.airlift.bootstrap:name=delta,type=lifecyclemanager"), row("io.trino.filesystem.s3:name=delta,type=s3filesystemstats"), row("io.trino.metastore.cache:name=delta,type=cachinghivemetastore"), row("io.trino.plugin.hive.metastore.thrift:name=delta,type=thrifthivemetastore"), diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java index 4c4f00ed2070..eda37f51851d 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeSelectCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_143; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_154; @@ -41,7 +40,7 @@ public class TestDeltaLakeSelectCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_133, DELTA_LAKE_DATABRICKS_143, DELTA_LAKE_DATABRICKS_154, DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testPartitionedSelectSpecialCharacters() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java index e7a7b48f69a5..3daf6ab68c3a 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeUpdateCompatibility.java @@ -24,7 +24,6 @@ import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_ISSUE; import static io.trino.tests.product.deltalake.util.DeltaLakeTestUtils.DATABRICKS_COMMUNICATION_FAILURE_MATCH; @@ -37,7 +36,7 @@ public class TestDeltaLakeUpdateCompatibility extends BaseTestDeltaLakeS3Storage { - @Test(groups = {DELTA_LAKE_DATABRICKS, DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testUpdatesFromDatabricks() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java index 078bec88fa9f..bfffd457c460 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeWriteDatabricksCompatibility.java @@ -37,7 +37,7 @@ import static io.trino.tempto.assertions.QueryAssert.assertQueryFailure; import static io.trino.testing.TestingNames.randomNameSuffix; import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; -import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_122; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS_133; import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; import static io.trino.tests.product.deltalake.S3ClientFactory.createS3Client; @@ -333,7 +333,7 @@ public void testInsertingIntoDatabricksTableWithAddedNotNullConstraint() } } - @Test(groups = {DELTA_LAKE_DATABRICKS_122, PROFILE_SPECIFIC_TESTS}) + @Test(groups = {DELTA_LAKE_DATABRICKS_133, PROFILE_SPECIFIC_TESTS}) @Flaky(issue = DATABRICKS_COMMUNICATION_FAILURE_ISSUE, match = DATABRICKS_COMMUNICATION_FAILURE_MATCH) public void testTrinoVacuumRemoveChangeDataFeedFiles() { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java index 2433a5ccd23d..854c4cd207da 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/util/DatabricksVersion.java @@ -23,7 +23,6 @@ public record DatabricksVersion(int majorVersion, int minorVersion) { public static final DatabricksVersion DATABRICKS_143_RUNTIME_VERSION = new DatabricksVersion(14, 3); public static final DatabricksVersion DATABRICKS_122_RUNTIME_VERSION = new DatabricksVersion(12, 2); - public static final DatabricksVersion DATABRICKS_113_RUNTIME_VERSION = new DatabricksVersion(11, 3); private static final Pattern DATABRICKS_VERSION_PATTERN = Pattern.compile("(\\d+)\\.(\\d+)"); @@ -32,11 +31,6 @@ public boolean isAtLeast(DatabricksVersion version) return compareTo(version) >= 0; } - public boolean isOlderThan(DatabricksVersion version) - { - return compareTo(version) < 0; - } - @Override public int compareTo(DatabricksVersion other) { diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveOnOrcLegacyDateCompatibility.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveOnOrcLegacyDateCompatibility.java new file mode 100644 index 000000000000..cea5ca741e16 --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestHiveOnOrcLegacyDateCompatibility.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.hive; + +import io.trino.tempto.ProductTest; +import io.trino.tempto.assertions.QueryAssert; +import org.testng.annotations.Test; + +import java.sql.Date; +import java.sql.Timestamp; + +import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.tests.product.TestGroups.HIVE4; +import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.utils.QueryExecutors.onHive; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; +import static java.lang.String.format; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHiveOnOrcLegacyDateCompatibility + extends ProductTest +{ + private static final String TRINO_CATALOG = "hive"; + private static final String SCHEMA = "default"; + + @Test(groups = {HIVE4, PROFILE_SPECIFIC_TESTS}) + public void testReadLegacyDateFromOrcWrittenByTrino() + { + String hiveTableName = "test_hive_orc_legacy_date_compatibility_%s".formatted(randomNameSuffix()); + String trinoTableName = format("%s.%s.%s", TRINO_CATALOG, SCHEMA, hiveTableName); + + try { + onTrino().executeQuery("CREATE TABLE %s (date_col date, t timestamp) WITH (format = 'ORC')".formatted(trinoTableName)); + onTrino().executeQuery(""" + INSERT INTO %s VALUES + (DATE '0002-01-01', TIMESTAMP '0002-01-01 00:00:00.123'), + (DATE '1500-01-01', TIMESTAMP '1500-01-01 00:00:00.123'), + (DATE '1582-10-04', TIMESTAMP '1582-10-04 00:00:00.123'), + (DATE '1582-11-04', TIMESTAMP '1582-11-04 00:00:00.123'), + (DATE '2000-02-29', TIMESTAMP '2000-02-29 00:00:00.123')""".formatted(trinoTableName)); + + QueryAssert.Row[] expectedRows = { + row(Date.valueOf("0002-01-01"), Timestamp.valueOf("0002-01-01 00:00:00.123")), + row(Date.valueOf("1500-01-01"), Timestamp.valueOf("1500-01-01 00:00:00.123")), + row(Date.valueOf("1582-10-04"), Timestamp.valueOf("1582-10-04 00:00:00.123")), + row(Date.valueOf("1582-11-04"), Timestamp.valueOf("1582-11-04 00:00:00.123")), + row(Date.valueOf("2000-02-29"), Timestamp.valueOf("2000-02-29 00:00:00.123"))}; + + assertThat(onTrino().executeQuery("SELECT date_col, t FROM " + trinoTableName)).containsOnly(expectedRows); + assertThat(onHive().executeQuery("SELECT date_col, t FROM " + hiveTableName)).containsOnly(expectedRows); + } + finally { + onTrino().executeQuery("DROP TABLE IF EXISTS " + trinoTableName); + } + } + + @Test(groups = {HIVE4, PROFILE_SPECIFIC_TESTS}) + public void testReadLegacyDateFromOrcWrittenByHive() + { + String hiveTableName = "test_hive_orc_legacy_date_compatibility_%s".formatted(randomNameSuffix()); + String trinoTableName = format("%s.%s.%s", TRINO_CATALOG, SCHEMA, hiveTableName); + + try { + onHive().executeQuery("CREATE TABLE %s (date_col date, t timestamp) STORED AS ORC".formatted(hiveTableName)); + onHive().executeQuery(""" + INSERT INTO %s VALUES + ('0002-01-01', '0002-01-01 00:00:00.123'), + ('1500-01-01', '1500-01-01 00:00:00.123'), + ('1582-10-04', '1582-10-04 00:00:00.123'), + ('1582-11-04', '1582-11-04 00:00:00.123'), + ('2000-02-29', '2000-02-29 00:00:00.123')""".formatted(hiveTableName)); + + QueryAssert.Row[] expectedRowsHive = { + row(Date.valueOf("0002-01-01"), Timestamp.valueOf("0002-01-01 00:00:00.123")), + row(Date.valueOf("1500-01-01"), Timestamp.valueOf("1500-01-01 00:00:00.123")), + row(Date.valueOf("1582-10-04"), Timestamp.valueOf("1582-10-04 00:00:00.123")), + row(Date.valueOf("1582-11-04"), Timestamp.valueOf("1582-11-04 00:00:00.123")), + row(Date.valueOf("2000-02-29"), Timestamp.valueOf("2000-02-29 00:00:00.123"))}; + assertThat(onHive().executeQuery("SELECT date_col, t FROM " + hiveTableName)).containsOnly(expectedRowsHive); + + // https://github.com/trinodb/trino/issues/26865 + QueryAssert.Row[] expectedRowsTrino = { + row(Date.valueOf("0001-12-30"), Timestamp.valueOf("0001-12-30 00:00:00.123")), + row(Date.valueOf("1500-01-10"), Timestamp.valueOf("1500-01-10 00:00:00.123")), + row(Date.valueOf("1582-10-24"), Timestamp.valueOf("1582-10-24 00:00:00.123")), + row(Date.valueOf("1582-11-04"), Timestamp.valueOf("1582-11-04 00:00:00.123")), + row(Date.valueOf("2000-02-29"), Timestamp.valueOf("2000-02-29 00:00:00.123"))}; + assertThat(onTrino().executeQuery("SELECT date_col, t FROM " + trinoTableName)).containsOnly(expectedRowsTrino); + } + finally { + onHive().executeQuery("DROP TABLE IF EXISTS " + trinoTableName); + } + } +} diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/loki/TestLoki.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/loki/TestLoki.java new file mode 100644 index 000000000000..2ef43636751c --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/loki/TestLoki.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.loki; + +import com.google.common.collect.ImmutableMap; +import io.github.jeschkies.loki.client.LokiClient; +import io.github.jeschkies.loki.client.LokiClientConfig; +import org.testng.annotations.Test; + +import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.time.format.DateTimeFormatter; + +import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.tests.product.TestGroups.LOKI; +import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; +import static java.time.ZoneOffset.UTC; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestLoki +{ + private static final DateTimeFormatter TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss.SSS'Z'").withZone(UTC); + + @Test(groups = {LOKI, PROFILE_SPECIFIC_TESTS}) + public void testQueryRange() + throws Exception + { + LokiClient client = new LokiClient(new LokiClientConfig(URI.create("http://loki:3100"), Duration.ofSeconds(10))); + + Instant start = Instant.now().minus(Duration.ofHours(3)); + Instant end = start.plus(Duration.ofHours(2)); + + client.pushLogLine("line 1", end.minus(Duration.ofMinutes(10)), ImmutableMap.of("test", "logs_query")); + client.pushLogLine("line 2", end.minus(Duration.ofMinutes(5)), ImmutableMap.of("test", "logs_query")); + client.pushLogLine("line 3", end.minus(Duration.ofMinutes(1)), ImmutableMap.of("test", "logs_query")); + client.flush(); + + assertThat(onTrino().executeQuery("SELECT value FROM TABLE(loki.system.query_range(" + + "'{test=\"logs_query\"}'," + + "TIMESTAMP '" + TIMESTAMP_FORMATTER.format(start) + "'," + + "TIMESTAMP '" + TIMESTAMP_FORMATTER.format(end) + "'))" + + "LIMIT 1")) + .containsOnly(row("line 1")); + } +} diff --git a/testing/trino-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result b/testing/trino-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result index 53df34f5523c..a3a047289c51 100644 --- a/testing/trino-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result +++ b/testing/trino-product-tests/src/main/resources/sql-tests/testcases/system/selectInformationSchemaColumns.result @@ -41,6 +41,7 @@ system| metadata| analyze_properties| description| varchar| YES| null| null| system| metadata| catalogs| catalog_name| varchar| YES| null| null| system| metadata| catalogs| connector_id| varchar| YES| null| null| system| metadata| catalogs| connector_name| varchar| YES| null| null| +system| metadata| catalogs| state| varchar| YES| null| null| system| metadata| column_properties| catalog_name| varchar| YES| null| null| system| metadata| column_properties| property_name| varchar| YES| null| null| system| metadata| column_properties| default_value| varchar| YES| null| null| diff --git a/testing/trino-server-dev/pom.xml b/testing/trino-server-dev/pom.xml index 4da7b9ed0084..fccd12c01cfb 100644 --- a/testing/trino-server-dev/pom.xml +++ b/testing/trino-server-dev/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -40,6 +40,11 @@ trino-main + + io.trino + trino-plugin-toolkit + + io.trino trino-spi diff --git a/testing/trino-server-dev/src/main/java/io/trino/server/DevelopmentServer.java b/testing/trino-server-dev/src/main/java/io/trino/server/DevelopmentServer.java index 48cc83e4c9de..efddca829bc1 100644 --- a/testing/trino-server-dev/src/main/java/io/trino/server/DevelopmentServer.java +++ b/testing/trino-server-dev/src/main/java/io/trino/server/DevelopmentServer.java @@ -18,8 +18,14 @@ import com.google.inject.Scopes; import io.trino.server.PluginManager.PluginsProvider; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; + import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.trino.plugin.base.ClosingBinder.closingBinder; public final class DevelopmentServer extends Server @@ -29,11 +35,24 @@ private DevelopmentServer() {} @Override protected Iterable getAdditionalModules() { - return ImmutableList.of(binder -> { - newOptionalBinder(binder, PluginsProvider.class).setBinding() - .to(DevelopmentPluginsProvider.class).in(Scopes.SINGLETON); - configBinder(binder).bindConfig(DevelopmentLoaderConfig.class); - }); + try { + Path pluginPath = Files.createTempDirectory("plugins"); + + return ImmutableList.of(binder -> { + newOptionalBinder(binder, PluginsProvider.class).setBinding() + .to(DevelopmentPluginsProvider.class).in(Scopes.SINGLETON); + configBinder(binder).bindConfig(DevelopmentLoaderConfig.class); + + // Use a temporary directory to satisfy configuration validation + configBinder(binder).bindConfigDefaults(ServerPluginsProviderConfig.class, config -> + config.setInstalledPluginsDirs(ImmutableList.of(pluginPath))); + + closingBinder(binder).registerCloseable(() -> Files.deleteIfExists(pluginPath)); + }); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } } public static void main(String[] args) diff --git a/testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh b/testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh index 3221785525e6..de80c559b247 100755 --- a/testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh +++ b/testing/trino-test-jdbc-compatibility-old-driver/bin/run_tests.sh @@ -8,7 +8,7 @@ maven_run_tests="${maven} clean test ${MAVEN_TEST:--B} -pl :trino-test-jdbc-comp "${maven}" -version -current_version=$(${maven} help:evaluate -Dexpression=project.version -q -DforceStdout) +current_version=$(${maven} help:evaluate -Dexpression=project.version -q -DforceStdout --raw-streams) previous_released_version=$((${current_version%-SNAPSHOT}-1)) first_tested_version=352 # test n-th version only @@ -18,8 +18,8 @@ echo "Current version: ${current_version}" echo "Testing every ${version_step}. version between ${first_tested_version} and ${previous_released_version}" # 404 was skipped -# 422-424 depend on the incompatible version of the open-telemetry semantic conventions used while invoking tests -tested_versions=$(seq "${first_tested_version}" ${version_step} "${previous_released_version}" | grep -vx '404\|42[234]') +# 422-424 and 442 depend on the incompatible version of the open-telemetry semantic conventions used while invoking tests +tested_versions=$(seq "${first_tested_version}" ${version_step} "${previous_released_version}" | grep -vx '404\|42[234]\|442') if (( (previous_released_version - first_tested_version) % version_step != 0 )); then tested_versions="${tested_versions} ${previous_released_version}" diff --git a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml index 5ad6dae06668..5014a28af673 100644 --- a/testing/trino-test-jdbc-compatibility-old-driver/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-driver/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -15,7 +15,7 @@ - 478-SNAPSHOT + 479-SNAPSHOT diff --git a/testing/trino-test-jdbc-compatibility-old-server/pom.xml b/testing/trino-test-jdbc-compatibility-old-server/pom.xml index 7bd56e33bc29..5f0c97e1e4d3 100644 --- a/testing/trino-test-jdbc-compatibility-old-server/pom.xml +++ b/testing/trino-test-jdbc-compatibility-old-server/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -90,7 +90,7 @@ org.testcontainers - trino + testcontainers-trino test diff --git a/testing/trino-test-jdbc-compatibility-old-server/src/test/java/io/trino/TestJdbcResultSetCompatibilityOldServer.java b/testing/trino-test-jdbc-compatibility-old-server/src/test/java/io/trino/TestJdbcResultSetCompatibilityOldServer.java index bc8f0a09da5d..427681eb3f9a 100644 --- a/testing/trino-test-jdbc-compatibility-old-server/src/test/java/io/trino/TestJdbcResultSetCompatibilityOldServer.java +++ b/testing/trino-test-jdbc-compatibility-old-server/src/test/java/io/trino/TestJdbcResultSetCompatibilityOldServer.java @@ -17,7 +17,7 @@ import com.google.common.io.Resources; import io.trino.jdbc.BaseTestJdbcResultSet; import org.testcontainers.DockerClientFactory; -import org.testcontainers.containers.TrinoContainer; +import org.testcontainers.trino.TrinoContainer; import org.testcontainers.utility.DockerImageName; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; diff --git a/testing/trino-testing-containers/pom.xml b/testing/trino-testing-containers/pom.xml index 1e2dad4b585a..c22ee0d6db83 100644 --- a/testing/trino-testing-containers/pom.xml +++ b/testing/trino-testing-containers/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Alluxio.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Alluxio.java new file mode 100644 index 000000000000..5032eac589b5 --- /dev/null +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Alluxio.java @@ -0,0 +1,95 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.testing.containers; + +import com.google.common.io.Closer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +import java.time.Duration; + +public class Alluxio + implements AutoCloseable +{ + private static final String IMAGE_NAME = "alluxio/alluxio:2.9.5"; + private static final DockerImageName ALLUXIO_IMAGE = DockerImageName.parse(IMAGE_NAME); + + private final Closer closer = Closer.create(); + + private final GenericContainer alluxioMaster; + private final GenericContainer alluxioWorker; + + public Alluxio() + { + this.alluxioMaster = createAlluxioMasterContainer(); + this.alluxioWorker = createAlluxioWorkerContainer(); + + closer.register(alluxioWorker::close); + closer.register(alluxioMaster::close); + } + + public void start() + { + alluxioMaster.start(); + alluxioWorker.start(); + } + + @Override + public void close() + throws Exception + { + closer.close(); + } + + private static GenericContainer createAlluxioMasterContainer() + { + return new GenericContainer<>(ALLUXIO_IMAGE).withCommand("master-only") + .withEnv("ALLUXIO_JAVA_OPTS", + "-Dalluxio.security.authentication.type=NOSASL " + + "-Dalluxio.master.hostname=localhost " + + "-Dalluxio.worker.hostname=localhost " + + "-Dalluxio.master.mount.table.root.ufs=/opt/alluxio/underFSStorage " + + "-Dalluxio.master.journal.type=NOOP " + + "-Dalluxio.security.authorization.permission.enabled=false " + + "-Dalluxio.security.authorization.plugins.enabled=false ") + .withNetworkMode("host") + .withAccessToHost(true) + .withStartupAttempts(10) + .withStartupTimeout(Duration.ofMinutes(10)) + .waitingFor(new LogMessageWaitStrategy() + .withRegEx(".*Primary started*\n") + .withStartupTimeout(Duration.ofMinutes(3))); + } + + private static GenericContainer createAlluxioWorkerContainer() + { + return new GenericContainer<>(ALLUXIO_IMAGE).withCommand("worker-only") + .withNetworkMode("host") + .withEnv("ALLUXIO_JAVA_OPTS", + "-Dalluxio.security.authentication.type=NOSASL " + + "-Dalluxio.worker.ramdisk.size=512MB " + + "-Dalluxio.worker.hostname=localhost " + + "-Dalluxio.worker.tieredstore.level0.alias=HDD " + + "-Dalluxio.worker.tieredstore.level0.dirs.path=/tmp " + + "-Dalluxio.master.hostname=localhost " + + "-Dalluxio.security.authorization.permission.enabled=false " + + "-Dalluxio.security.authorization.plugins.enabled=false ") + .withAccessToHost(true) + .withStartupAttempts(10) + .withStartupTimeout(Duration.ofMinutes(10)) + .waitingFor(Wait.forLogMessage(".*Alluxio worker started.*\n", 1)); + } +} diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Minio.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Minio.java index 02f7174a7625..5950fa0a2ba2 100644 --- a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Minio.java +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/Minio.java @@ -23,6 +23,7 @@ import io.airlift.log.Logger; import io.trino.testing.minio.MinioClient; import org.testcontainers.containers.Network; +import org.testcontainers.utility.DockerImageName; import java.io.IOException; import java.io.UncheckedIOException; @@ -41,7 +42,8 @@ public class Minio { private static final Logger log = Logger.get(Minio.class); - public static final String DEFAULT_IMAGE = "minio/minio:RELEASE.2024-12-18T13-15-44Z"; + public static final String DEFAULT_IMAGE = DockerImageName.parse("cgr.dev/chainguard/minio@sha256:66bd82c8fe5e75868ae7d0b2e102d9a0dcf971b270a41bd060a9e6a643476ff8") + .asCanonicalNameString(); public static final String DEFAULT_HOST_NAME = "minio"; public static final int MINIO_API_PORT = 4566; @@ -80,6 +82,7 @@ private Minio( protected void setupContainer() { super.setupContainer(); + withCreateContainerModifier(cmd -> cmd.withUser("root")); // Required to create buckets externally withRunCommand( ImmutableList.of( "server", diff --git a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/junit/ReportLeakedContainers.java b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/junit/ReportLeakedContainers.java index 61904e4e436c..f5a3284e629e 100644 --- a/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/junit/ReportLeakedContainers.java +++ b/testing/trino-testing-containers/src/main/java/io/trino/testing/containers/junit/ReportLeakedContainers.java @@ -85,6 +85,8 @@ public void testPlanExecutionFinished(TestPlan testPlan) .withStatusFilter(List.of("created", "restarting", "running", "paused")) .exec() .stream() + // testcontainers/sshd is implicitly started by testcontainers and we trust the library to stop if when no longer needed + .filter(container -> !container.getImage().startsWith("testcontainers/sshd:")) .filter(container -> !ignoredIds.contains(container.getId())) .collect(toImmutableList()); diff --git a/testing/trino-testing-kafka/pom.xml b/testing/trino-testing-kafka/pom.xml index 0dc74a70aa2d..e9d88861a3ff 100644 --- a/testing/trino-testing-kafka/pom.xml +++ b/testing/trino-testing-kafka/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -44,11 +44,6 @@ kafka-clients - - org.testcontainers - kafka - - org.testcontainers testcontainers @@ -60,6 +55,11 @@ + + org.testcontainers + testcontainers-kafka + + io.airlift junit-extensions diff --git a/testing/trino-testing-resources/pom.xml b/testing/trino-testing-resources/pom.xml index f51081d4d9ba..b6925e37c0ef 100644 --- a/testing/trino-testing-resources/pom.xml +++ b/testing/trino-testing-resources/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing-services/pom.xml b/testing/trino-testing-services/pom.xml index 903b7bc97a47..680fde2df234 100644 --- a/testing/trino-testing-services/pom.xml +++ b/testing/trino-testing-services/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing/pom.xml b/testing/trino-testing/pom.xml index 8cfff958293f..520c9485e0f0 100644 --- a/testing/trino-testing/pom.xml +++ b/testing/trino-testing/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml diff --git a/testing/trino-testing/src/main/java/io/trino/testing/AbstractDistributedEngineOnlyQueries.java b/testing/trino-testing/src/main/java/io/trino/testing/AbstractDistributedEngineOnlyQueries.java index eee80f11535c..134ed0a875e3 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/AbstractDistributedEngineOnlyQueries.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/AbstractDistributedEngineOnlyQueries.java @@ -34,9 +34,11 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; import static io.trino.SystemSessionProperties.ENABLE_LARGE_DYNAMIC_FILTERS; +import static io.trino.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static io.trino.execution.QueryState.RUNNING; import static io.trino.sql.planner.OptimizerConfig.JoinDistributionType.BROADCAST; import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.testing.TestingSession.testSessionBuilder; import static io.trino.testing.assertions.Assert.assertEventually; import static java.lang.String.format; import static java.util.concurrent.Executors.newCachedThreadPool; @@ -403,4 +405,31 @@ public void testRowConstructorColumnLimit() @Language("SQL") String query = "SELECT row(" + rowFields + ") FROM (select * from tpch.tiny.orders limit 1) t(" + colNames + ")"; assertThat(getQueryRunner().execute(query).getOnlyValue()).isNotNull(); } + + @Test + public void testFragmentPartitioningWithValues() + { + // Test fragment with empty VALUES and a table scan + Session broadcastJonSession = testSessionBuilder(getSession()) + .setSystemProperty(JOIN_DISTRIBUTION_TYPE, "BROADCAST") + .build(); + assertUpdate("CREATE TABLE t1 (a bigint)"); + assertUpdate("INSERT INTO t1 VALUES (1), (2), (3)", 3); + assertUpdate("CREATE TABLE t2 (a bigint, b varchar)"); + assertThat(query( + broadcastJonSession, + """ + WITH t3 AS ( + SELECT a, CAST(null AS varchar) b FROM t1 + UNION ALL + SELECT a, b FROM t2) + SELECT * FROM t3 WHERE b IN (SELECT b FROM t2) + """)) + .returnsEmptyResult(); + assertUpdate("DROP TABLE t1"); + assertUpdate("DROP TABLE t2"); + + // Test fragment with empty VALUES and no table scans + assertQuery("SELECT * FROM (SELECT 2 WHERE FALSE)"); + } } diff --git a/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java b/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java index dccd16176243..209bfe699933 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/AbstractTestQueryFramework.java @@ -232,7 +232,7 @@ private void checkTasksDone() List taskInfos = taskManager.getAllTaskInfo(); for (TaskInfo taskInfo : taskInfos) { TaskId taskId = taskInfo.taskStatus().getTaskId(); - QueryId queryId = taskId.getQueryId(); + QueryId queryId = taskId.queryId(); TaskState taskState = taskInfo.taskStatus().getState(); if (!taskState.isDone()) { try { @@ -431,6 +431,11 @@ protected void assertUpdate(Session session, @Language("SQL") String sql, long c QueryAssertions.assertUpdate(queryRunner, session, sql, OptionalLong.of(count), Optional.of(planAssertion)); } + protected void assertUpdate(Session session, @Language("SQL") String sql, Consumer planAssertion) + { + QueryAssertions.assertUpdate(queryRunner, session, sql, OptionalLong.empty(), Optional.of(planAssertion)); + } + protected void assertQuerySucceeds(@Language("SQL") String sql) { assertQuerySucceeds(getSession(), sql); diff --git a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java index 1568dbca21f2..32f224ca5e56 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorSmokeTest.java @@ -282,8 +282,7 @@ public void testDeleteAllDataFromTable() public void testRowLevelDelete() { assumeTrue(hasBehavior(SUPPORTS_CREATE_TABLE) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = newTrinoTable("test_row_delete", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_row_level_delete", "AS SELECT * FROM region")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE regionkey = 2", 1); assertThat(query("SELECT * FROM " + table.getName() + " WHERE regionkey = 2")) .returnsEmptyResult(); diff --git a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java index 5a25189aa369..0ffeb7beafbd 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/BaseConnectorTest.java @@ -30,18 +30,24 @@ import io.trino.metadata.FunctionManager; import io.trino.metadata.Metadata; import io.trino.metadata.QualifiedObjectName; +import io.trino.plugin.base.metrics.DistributionSnapshot; +import io.trino.plugin.base.metrics.LongCount; import io.trino.server.BasicQueryInfo; +import io.trino.spi.QueryId; import io.trino.spi.connector.ConnectorSession; import io.trino.spi.connector.MaterializedViewFreshness; +import io.trino.spi.metrics.Metrics; import io.trino.spi.security.Identity; import io.trino.sql.planner.OptimizerConfig.JoinDistributionType; import io.trino.sql.planner.Plan; import io.trino.sql.planner.assertions.PlanMatchPattern; import io.trino.sql.planner.plan.AggregationNode; +import io.trino.sql.planner.plan.FilterNode; import io.trino.sql.planner.plan.LimitNode; import io.trino.sql.planner.plan.OutputNode; import io.trino.sql.planner.plan.ProjectNode; import io.trino.sql.planner.plan.TableScanNode; +import io.trino.sql.planner.plan.TopNNode; import io.trino.sql.query.QueryAssertions.QueryAssert; import io.trino.testing.QueryRunner.MaterializedResultWithPlan; import io.trino.testing.assertions.TrinoExceptionAssert; @@ -61,6 +67,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.OptionalInt; import java.util.concurrent.ConcurrentHashMap; @@ -95,6 +102,7 @@ import static io.trino.spi.type.VarcharType.VARCHAR; import static io.trino.sql.planner.assertions.PlanMatchPattern.anyTree; import static io.trino.sql.planner.assertions.PlanMatchPattern.node; +import static io.trino.sql.planner.assertions.PlanMatchPattern.project; import static io.trino.sql.planner.assertions.PlanMatchPattern.tableScan; import static io.trino.sql.planner.optimizations.PlanNodeSearcher.searchFrom; import static io.trino.sql.planner.planprinter.PlanPrinter.textLogicalPlan; @@ -108,6 +116,7 @@ import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_ADD_COLUMN_WITH_POSITION; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_ADD_FIELD; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_ADD_FIELD_IN_ARRAY; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_AGGREGATION_PUSHDOWN; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_ARRAY; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_COMMENT_ON_COLUMN; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_COMMENT_ON_MATERIALIZED_VIEW_COLUMN; @@ -134,12 +143,15 @@ import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_DROP_NOT_NULL_CONSTRAINT; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_DROP_SCHEMA_CASCADE; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_INSERT; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_LIMIT_PUSHDOWN; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_MAP_TYPE; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_MATERIALIZED_VIEW_FRESHNESS_FROM_BASE_TABLES; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_MERGE; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_MULTI_STATEMENT_WRITES; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_NEGATIVE_DATE; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_NOT_NULL_CONSTRAINT; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_PREDICATE_PUSHDOWN; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_REFRESH_VIEW; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_RENAME_COLUMN; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_RENAME_FIELD; @@ -157,6 +169,7 @@ import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_SET_FIELD_TYPE_IN_ARRAY; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_SET_FIELD_TYPE_IN_MAP; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_TOPN_PUSHDOWN; +import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_TRUNCATE; import static io.trino.testing.TestingConnectorBehavior.SUPPORTS_UPDATE; import static io.trino.testing.TestingNames.randomNameSuffix; @@ -415,6 +428,144 @@ public void testVarcharCharComparison() } } + @Test + public void testLimitPushdown() + { + if (!hasBehavior(SUPPORTS_LIMIT_PUSHDOWN)) { + assertThat(query("SELECT name FROM nation LIMIT 30")).isNotFullyPushedDown(LimitNode.class); // Use high limit for result determinism + return; + } + + assertThat(query("SELECT name FROM nation LIMIT 30")).isFullyPushedDown(); // Use high limit for result determinism + assertThat(query("SELECT name FROM nation LIMIT 3")).skipResultsCorrectnessCheckForPushdown().isFullyPushedDown(); + + PlanMatchPattern filterOverTableScan = node(FilterNode.class, node(TableScanNode.class)); + // with filter over numeric column + assertConditionallyPushedDown( + getSession(), + "SELECT name FROM nation WHERE regionkey = 3 LIMIT 5", + hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN), + filterOverTableScan); + + // with filter over varchar column + assertConditionallyPushedDown( + getSession(), + "SELECT name FROM nation WHERE name < 'EEE' LIMIT 5", + hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY), + filterOverTableScan); + + // with aggregation + PlanMatchPattern aggregationOverTableScan = node(AggregationNode.class, anyTree(node(TableScanNode.class))); + assertConditionallyPushedDown( + getSession(), + "SELECT max(regionkey) FROM nation LIMIT 5", // global aggregation, LIMIT removed + hasBehavior(SUPPORTS_AGGREGATION_PUSHDOWN), + aggregationOverTableScan); + assertConditionallyPushedDown( + getSession(), + "SELECT regionkey, max(nationkey) FROM nation GROUP BY regionkey LIMIT 5", + hasBehavior(SUPPORTS_AGGREGATION_PUSHDOWN), + aggregationOverTableScan); + + // with aggregation and filter over numeric column + if (hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN)) { + assertConditionallyPushedDown( + getSession(), + "SELECT regionkey, count(*) FROM nation WHERE nationkey < 5 GROUP BY regionkey LIMIT 3", + hasBehavior(SUPPORTS_AGGREGATION_PUSHDOWN), + aggregationOverTableScan); + } + + // with aggregation and filter over varchar column + if (hasBehavior(SUPPORTS_PREDICATE_PUSHDOWN_WITH_VARCHAR_INEQUALITY)) { + assertConditionallyPushedDown( + getSession(), + "SELECT regionkey, count(*) FROM nation WHERE name < 'EGYPT' GROUP BY regionkey LIMIT 3", + hasBehavior(SUPPORTS_AGGREGATION_PUSHDOWN), + aggregationOverTableScan); + } + + // with TopN over numeric column + PlanMatchPattern topnOverTableScan = project(node(TopNNode.class, anyTree(node(TableScanNode.class)))); + assertConditionallyPushedDown( + getSession(), + "SELECT * FROM (SELECT regionkey FROM nation ORDER BY nationkey ASC LIMIT 10) LIMIT 5", + hasBehavior(SUPPORTS_TOPN_PUSHDOWN), + topnOverTableScan); + // with TopN over varchar column + assertConditionallyPushedDown( + getSession(), + "SELECT * FROM (SELECT regionkey FROM nation ORDER BY name ASC LIMIT 10) LIMIT 5", + hasBehavior(SUPPORTS_TOPN_PUSHDOWN_WITH_VARCHAR), + topnOverTableScan); + } + + @Test + public void testTopNPushdown() + { + if (!hasBehavior(SUPPORTS_TOPN_PUSHDOWN)) { + assertThat(query("SELECT orderkey FROM orders ORDER BY orderkey LIMIT 10")) + .ordered() + .isNotFullyPushedDown(TopNNode.class); + return; + } + + assertThat(query("SELECT orderkey FROM orders ORDER BY orderkey LIMIT 10")) + .ordered() + .isFullyPushedDown(); + + assertThat(query("SELECT orderkey FROM orders ORDER BY orderkey DESC LIMIT 10")) + .ordered() + .isFullyPushedDown(); + + // multiple sort columns with different orders + assertThat(query("SELECT * FROM orders ORDER BY shippriority DESC, totalprice ASC LIMIT 10")) + .ordered() + .isFullyPushedDown(); + + // TopN over aggregation column + if (hasBehavior(SUPPORTS_AGGREGATION_PUSHDOWN)) { + assertThat(query("SELECT sum(totalprice) AS total FROM orders GROUP BY custkey ORDER BY total DESC LIMIT 10")) + .ordered() + .isFullyPushedDown(); + } + + // TopN over TopN + assertThat(query("SELECT orderkey, totalprice FROM (SELECT orderkey, totalprice FROM orders ORDER BY 1, 2 LIMIT 10) ORDER BY 2, 1 LIMIT 5")) + .ordered() + .isFullyPushedDown(); + + assertThat(query("" + + "SELECT orderkey, totalprice " + + "FROM (SELECT orderkey, totalprice FROM (SELECT orderkey, totalprice FROM orders ORDER BY 1, 2 LIMIT 10) " + + "ORDER BY 2, 1 LIMIT 5) ORDER BY 1, 2 LIMIT 3")) + .ordered() + .isFullyPushedDown(); + + // TopN over limit - use high limit for deterministic result + assertThat(query("SELECT orderkey, totalprice FROM (SELECT orderkey, totalprice FROM orders LIMIT 15000) ORDER BY totalprice ASC LIMIT 5")) + .ordered() + .isFullyPushedDown(); + + // TopN over limit with filter + assertThat(query("" + + "SELECT orderkey, totalprice " + + "FROM (SELECT orderkey, totalprice FROM orders WHERE orderdate = DATE '1995-09-16' LIMIT 20) " + + "ORDER BY totalprice ASC LIMIT 5")) + .ordered() + .isFullyPushedDown(); + + // TopN over aggregation with filter + if (hasBehavior(SUPPORTS_AGGREGATION_PUSHDOWN)) { + assertThat(query("" + + "SELECT * " + + "FROM (SELECT SUM(totalprice) as sum, custkey AS total FROM orders GROUP BY custkey HAVING COUNT(*) > 3) " + + "ORDER BY sum DESC LIMIT 10")) + .ordered() + .isFullyPushedDown(); + } + } + @Test public void testAggregation() { @@ -3560,8 +3711,7 @@ public void testCreateTable() assertQueryFails("CREATE TABLE " + tableName + " (a bad_type)", ".* Unknown type 'bad_type' for column 'a'"); assertThat(getQueryRunner().tableExists(getSession(), tableName)).isFalse(); - // TODO (https://github.com/trinodb/trino/issues/5901) revert to longer name when Oracle version is updated - tableName = "test_cr_not_exists_" + randomNameSuffix(); + tableName = "test_create_table_not_exists_" + randomNameSuffix(); assertUpdate("CREATE TABLE " + tableName + " (a bigint, b varchar(50), c double)"); assertThat(getQueryRunner().tableExists(getSession(), tableName)).isTrue(); assertTableColumnNames(tableName, "a", "b", "c"); @@ -3863,7 +4013,7 @@ public void testCreateTableWithLongColumnName() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE)); - String tableName = "test_long_column" + randomNameSuffix(); + String tableNameWithValidColumnLength = "test_long_column" + randomNameSuffix(); String baseColumnName = "col"; int maxLength = maxColumnNameLength() @@ -3871,18 +4021,19 @@ public void testCreateTableWithLongColumnName() .orElse(65536 + 5); String validColumnName = baseColumnName + "z".repeat(maxLength - baseColumnName.length()); - assertUpdate("CREATE TABLE " + tableName + " (" + validColumnName + " bigint)"); - assertThat(columnExists(tableName, validColumnName)).isTrue(); - assertUpdate("DROP TABLE " + tableName); + assertUpdate("CREATE TABLE " + tableNameWithValidColumnLength + " (" + validColumnName + " bigint)"); + assertThat(columnExists(tableNameWithValidColumnLength, validColumnName)).isTrue(); + assertUpdate("DROP TABLE " + tableNameWithValidColumnLength); if (maxColumnNameLength().isEmpty()) { return; } + String tableNameWithInvalidColumnLength = "test_long_column" + randomNameSuffix(); String invalidColumnName = validColumnName + "z"; - assertThatThrownBy(() -> assertUpdate("CREATE TABLE " + tableName + " (" + invalidColumnName + " bigint)")) + assertThatThrownBy(() -> assertUpdate("CREATE TABLE " + tableNameWithInvalidColumnLength + " (" + invalidColumnName + " bigint)")) .satisfies(this::verifyColumnNameLengthFailurePermissible); - assertThat(getQueryRunner().tableExists(getSession(), tableName)).isFalse(); + assertThat(getQueryRunner().tableExists(getSession(), tableNameWithInvalidColumnLength)).isFalse(); } // TODO: Add test for CREATE TABLE AS SELECT with long column name @@ -4440,8 +4591,7 @@ protected void testCommentColumnName(String columnName, boolean delimited) { String nameInSql = toColumnNameInSql(columnName, delimited); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = newTrinoTable("test_comment_column", "(" + nameInSql + " integer)")) { + try (TestTable table = newTrinoTable("test_comment_column_name", "(" + nameInSql + " integer)")) { assertUpdate("COMMENT ON COLUMN " + table.getName() + "." + nameInSql + " IS 'test comment'"); assertThat(getColumnComment(table.getName(), columnName.replace("'", "''").toLowerCase(ENGLISH))).isEqualTo("test comment"); } @@ -4910,8 +5060,7 @@ public void testDeleteWithComplexPredicate() { skipTestUnless(hasBehavior(SUPPORTS_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_complex_", "AS SELECT * FROM nation", "nationkey")) { + try (TestTable table = createTestTableForWrites("test_delete_with_complex_predicate_", "AS SELECT * FROM nation", "nationkey")) { // delete half the table, then delete the rest assertUpdate("DELETE FROM " + table.getName() + " WHERE nationkey % 2 = 0", "SELECT count(*) FROM nation WHERE nationkey % 2 = 0"); assertQuery("SELECT * FROM " + table.getName(), "SELECT * FROM nation WHERE nationkey % 2 <> 0"); @@ -4929,8 +5078,7 @@ public void testDeleteWithSubquery() // TODO (https://github.com/trinodb/trino/issues/13210) Migrate these tests to AbstractTestEngineOnlyQueries skipTestUnless(hasBehavior(SUPPORTS_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_subquery", "AS SELECT * FROM nation", "nationkey")) { + try (TestTable table = createTestTableForWrites("test_delete_with_subquery", "AS SELECT * FROM nation", "nationkey")) { // delete using a subquery assertUpdate("DELETE FROM " + table.getName() + " WHERE regionkey IN (SELECT regionkey FROM region WHERE name LIKE 'A%')", 15); assertQuery( @@ -4938,8 +5086,7 @@ public void testDeleteWithSubquery() "SELECT * FROM nation WHERE regionkey IN (SELECT regionkey FROM region WHERE name NOT LIKE 'A%')"); } - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_subquery", "AS SELECT * FROM orders", "orderkey")) { + try (TestTable table = createTestTableForWrites("test_delete_with_subquery", "AS SELECT * FROM orders", "orderkey")) { // delete using a scalar and EXISTS subquery assertUpdate("DELETE FROM " + table.getName() + " WHERE orderkey = (SELECT orderkey FROM orders ORDER BY orderkey LIMIT 1)", 1); assertUpdate("DELETE FROM " + table.getName() + " WHERE orderkey = (SELECT orderkey FROM orders WHERE false)", 0); @@ -4992,8 +5139,7 @@ public void testDeleteWithSemiJoin() { skipTestUnless(hasBehavior(SUPPORTS_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_semijoin", "AS SELECT * FROM nation", "nationkey")) { + try (TestTable table = createTestTableForWrites("test_delete_with_semijoin", "AS SELECT * FROM nation", "nationkey")) { // delete with multiple SemiJoin assertUpdate( "DELETE FROM " + table.getName() + " " + @@ -5007,8 +5153,7 @@ public void testDeleteWithSemiJoin() " OR regionkey IN (SELECT regionkey FROM region WHERE length(comment) >= 50)"); } - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = createTestTableForWrites("test_delete_semijoin", "AS SELECT * FROM orders", "orderkey")) { + try (TestTable table = createTestTableForWrites("test_delete_with_semijoin", "AS SELECT * FROM orders", "orderkey")) { // delete with SemiJoin null handling assertUpdate( "DELETE FROM " + table.getName() + "\n" + @@ -5076,8 +5221,7 @@ public void testDeleteAllDataFromTable() public void testRowLevelDelete() { skipTestUnless(hasBehavior(SUPPORTS_CREATE_TABLE_WITH_DATA) && hasBehavior(SUPPORTS_ROW_LEVEL_DELETE)); - // TODO (https://github.com/trinodb/trino/issues/5901) Use longer table name once Oracle version is updated - try (TestTable table = newTrinoTable("test_row_delete", "AS SELECT * FROM region")) { + try (TestTable table = newTrinoTable("test_row_level_delete", "AS SELECT * FROM region")) { assertUpdate("DELETE FROM " + table.getName() + " WHERE regionkey = 2", 1); assertQuery("SELECT count(*) FROM " + table.getName(), "VALUES 4"); } @@ -7481,6 +7625,36 @@ protected Consumer assertPartialLimitWithPreSortedInputsCount(Session sess }; } + protected Map getCatalogMetadataMetrics(QueryId queryId) + { + try { + return getDistributedQueryRunner().getCoordinator() + .getQueryManager() + .getFullQueryInfo(queryId) + .getQueryStats() + .getCatalogMetadataMetrics(); + } + catch (NoSuchElementException e) { + throw new RuntimeException("Couldn't find operator summary, probably due to query statistic collection error", e); + } + } + + protected static void assertDistributionMetricExists(Map metrics, String catalog, String metricKey) + { + assertThat(metrics).containsKey(catalog); + assertThat(metrics.get(catalog).getMetrics()).isNotEmpty(); + assertThat(metrics.get(catalog).getMetrics()).containsKey(metricKey); + assertThat(((DistributionSnapshot) metrics.get(catalog).getMetrics().get(metricKey)).total()).isGreaterThan(0); + } + + protected static void assertCountMetricExists(Map metrics, String catalog, String metricKey) + { + assertThat(metrics).containsKey(catalog); + assertThat(metrics.get(catalog).getMetrics()).isNotEmpty(); + assertThat(metrics.get(catalog).getMetrics()).containsKey(metricKey); + assertThat(((LongCount) metrics.get(catalog).getMetrics().get(metricKey)).getTotal()).isGreaterThan(0); + } + protected void withMockTableListing(String forSchema, Function> listing, Runnable closure) { requireNonNull(forSchema, "forSchema is null"); @@ -7575,4 +7749,17 @@ public String toString() ":" + sampleValueLiteral.replaceAll("[^a-zA-Z0-9_-]", ""); } } + + protected QueryAssert assertConditionallyPushedDown( + Session session, + @Language("SQL") String query, + boolean condition, + PlanMatchPattern otherwiseExpected) + { + QueryAssert queryAssert = assertThat(query(session, query)); + if (condition) { + return queryAssert.isFullyPushedDown(); + } + return queryAssert.isNotFullyPushedDown(otherwiseExpected); + } } diff --git a/testing/trino-testing/src/main/java/io/trino/testing/BaseFailureRecoveryTest.java b/testing/trino-testing/src/main/java/io/trino/testing/BaseFailureRecoveryTest.java index 189eeaba63d5..d2270bc69c24 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/BaseFailureRecoveryTest.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/BaseFailureRecoveryTest.java @@ -613,12 +613,12 @@ private ExecutionResult execute(Session session, String query, Optional String queryId = null; try { resultWithPlan = getDistributedQueryRunner().executeWithPlan(withTraceToken(session, traceToken), resolveTableName(query, tableName)); - queryId = resultWithPlan.queryId().getId(); + queryId = resultWithPlan.queryId().id(); } catch (RuntimeException e) { failure = e; if (e instanceof QueryFailedException queryFailedException) { - queryId = queryFailedException.getQueryId().getId(); + queryId = queryFailedException.getQueryId().id(); } } diff --git a/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java b/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java index a015a1e52f99..659f7e71ecac 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/DistributedQueryRunner.java @@ -30,6 +30,7 @@ import io.trino.Session.SessionBuilder; import io.trino.client.ClientSession; import io.trino.client.StatementClient; +import io.trino.connector.ConnectorServicesProvider; import io.trino.connector.CoordinatorDynamicCatalogManager; import io.trino.cost.StatsCalculator; import io.trino.execution.FailureInjector.InjectedFailureType; @@ -997,6 +998,7 @@ public void queryCompleted(QueryCompletedEvent queryCompletedEvent) closeAllSuppress(e, queryRunner); throw e; } + queryRunner.getCoordinator().getInstance(Key.get(ConnectorServicesProvider.class)).loadInitialCatalogs(); return queryRunner; } diff --git a/testing/trino-testing/src/main/java/io/trino/testing/QueryAssertions.java b/testing/trino-testing/src/main/java/io/trino/testing/QueryAssertions.java index 81f620546ef3..b514762dc701 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/QueryAssertions.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/QueryAssertions.java @@ -336,7 +336,7 @@ private static void assertDistributedQuery( List actualRows = actualResults.getMaterializedRows(); List expectedRows = expectedResults.getMaterializedRows(); - if (compareUpdate) { + if (compareUpdate && !actualResults.getUpdateType().equals(Optional.of("ALTER TABLE EXECUTE"))) { if (actualResults.getUpdateType().isEmpty()) { fail("update type not present for query " + queryId + ": \n" + actual); } @@ -457,7 +457,7 @@ protected static void assertQueryFailsEventually(QueryRunner queryRunner, Sessio assertEventually(timeout, () -> assertQueryFails(queryRunner, session, sql, expectedMessageRegExp)); } - protected static void assertQueryFails(QueryRunner queryRunner, Session session, @Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp) + public static void assertQueryFails(QueryRunner queryRunner, Session session, @Language("SQL") String sql, @Language("RegExp") String expectedMessageRegExp) { try { MaterializedResultWithPlan resultWithPlan = queryRunner.executeWithPlan(session, sql); @@ -470,7 +470,7 @@ protected static void assertQueryFails(QueryRunner queryRunner, Session session, } } - protected static void assertQueryReturnsEmptyResult(QueryRunner queryRunner, Session session, @Language("SQL") String sql) + public static void assertQueryReturnsEmptyResult(QueryRunner queryRunner, Session session, @Language("SQL") String sql) { QueryId queryId = null; try { diff --git a/testing/trino-testing/src/main/java/io/trino/testing/tpch/TpchScaledTable.java b/testing/trino-testing/src/main/java/io/trino/testing/tpch/TpchScaledTable.java index bce141daeb76..1b588042b783 100644 --- a/testing/trino-testing/src/main/java/io/trino/testing/tpch/TpchScaledTable.java +++ b/testing/trino-testing/src/main/java/io/trino/testing/tpch/TpchScaledTable.java @@ -55,6 +55,6 @@ public boolean equals(Object obj) } TpchScaledTable other = (TpchScaledTable) obj; return Objects.equals(this.tableName, other.tableName) && - this.scaleFactor == other.scaleFactor; + this.scaleFactor == other.scaleFactor; } } diff --git a/testing/trino-tests/pom.xml b/testing/trino-tests/pom.xml index 47b98958df7f..79bd613ced7c 100644 --- a/testing/trino-tests/pom.xml +++ b/testing/trino-tests/pom.xml @@ -5,7 +5,7 @@ io.trino trino-root - 478-SNAPSHOT + 479-SNAPSHOT ../../pom.xml @@ -384,13 +384,13 @@ org.testcontainers - localstack + testcontainers test org.testcontainers - testcontainers + testcontainers-localstack test @@ -439,6 +439,20 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + + + + com.amazonaws:*:* + + + + + org.basepom.maven duplicate-finder-maven-plugin diff --git a/testing/trino-tests/src/test/java/io/trino/connector/TestDynamicCatalogs.java b/testing/trino-tests/src/test/java/io/trino/connector/TestDynamicCatalogs.java new file mode 100644 index 000000000000..849aa82c06b0 --- /dev/null +++ b/testing/trino-tests/src/test/java/io/trino/connector/TestDynamicCatalogs.java @@ -0,0 +1,243 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.connector; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.Session; +import io.trino.plugin.memory.MemoryPlugin; +import io.trino.server.ServerConfig; +import io.trino.spi.catalog.CatalogName; +import io.trino.spi.catalog.CatalogProperties; +import io.trino.spi.catalog.CatalogStore; +import io.trino.spi.catalog.CatalogStoreFactory; +import io.trino.spi.connector.CatalogVersion; +import io.trino.spi.connector.ConnectorName; +import io.trino.testing.DistributedQueryRunner; +import io.trino.testing.H2QueryRunner; +import io.trino.testing.QueryRunner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.connector.FileCatalogStore.computeCatalogVersion; +import static io.trino.testing.QueryAssertions.assertQuery; +import static io.trino.testing.QueryAssertions.assertQueryFails; +import static io.trino.testing.QueryAssertions.assertQueryReturnsEmptyResult; +import static io.trino.testing.QueryAssertions.assertUpdate; +import static io.trino.testing.TestingNames.randomNameSuffix; +import static io.trino.testing.TestingSession.testSession; +import static java.util.Objects.requireNonNull; +import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; + +@Execution(SAME_THREAD) +public class TestDynamicCatalogs +{ + private static final String BROKEN_CATALOG = "broken_catalog"; + private static final String PREPOPULATED_CATALOG = "prepopulated_catalog"; + private static final CatalogName BROKEN_CATALOG_NAME = new CatalogName(BROKEN_CATALOG); + private static final CatalogName PREPOPULATED_CATALOG_NAME = new CatalogName(PREPOPULATED_CATALOG); + private static final ConnectorName MEMORY_CONNECTOR_NAME = new ConnectorName("memory"); + + @Test + public void testNewHealthyCatalog() + throws Exception + { + String catalogName = "new_catalog" + randomNameSuffix(); + Session session = testSession(); + QueryRunner queryRunner = DistributedQueryRunner.builder(session) + .setWorkerCount(0) + .build(); + queryRunner.installPlugin(new MemoryPlugin()); + queryRunner.createCatalog("healthy_catalog", "memory", ImmutableMap.of("memory.max-data-per-node", "128MB")); + H2QueryRunner h2QueryRunner = new H2QueryRunner(); + + assertQuery(queryRunner, session, "SHOW CATALOGS", h2QueryRunner, "VALUES 'healthy_catalog', 'system'", false, false); + + assertUpdate(queryRunner, session, "CREATE CATALOG %s USING memory WITH (\"memory.max-data-per-node\" = '128MB')".formatted(catalogName), OptionalLong.empty(), Optional.empty()); + assertQuery(queryRunner, session, "SHOW CATALOGS", h2QueryRunner, "VALUES 'healthy_catalog', '" + catalogName + "', 'system'", false, false); + + assertUpdate(queryRunner, session, "CREATE TABLE %s.default.test_table (age INT)".formatted(catalogName), OptionalLong.empty(), Optional.empty()); + assertUpdate(queryRunner, session, "INSERT INTO %s.default.test_table VALUES (10)".formatted(catalogName), OptionalLong.of(1), Optional.empty()); + assertQuery(queryRunner, session, "SELECT * FROM %s.default.test_table".formatted(catalogName), h2QueryRunner, "VALUES (10)", false, false); + + assertUpdate(queryRunner, session, "DROP CATALOG " + catalogName, OptionalLong.empty(), Optional.empty()); + assertQuery(queryRunner, session, "SHOW CATALOGS", h2QueryRunner, "VALUES 'healthy_catalog', 'system'", false, false); + } + + @Test + public void testPrepopulatedUnhealthyCatalog() + throws Exception + { + Session session = testSession(); + ImmutableMap properties = ImmutableMap.of("non_existing", "false"); + QueryRunner queryRunner = DistributedQueryRunner.builder(session) + .setAdditionalModule(new TestCatalogStoreModule(ImmutableMap.of(BROKEN_CATALOG_NAME, new CatalogProperties( + BROKEN_CATALOG_NAME, + computeCatalogVersion(BROKEN_CATALOG_NAME, MEMORY_CONNECTOR_NAME, properties), + MEMORY_CONNECTOR_NAME, + properties)))) + .setAdditionalSetup(runner -> runner.installPlugin(new MemoryPlugin())) + .setCoordinatorProperties(ImmutableMap.of("catalog.store", "prepopulated_memory")) + .setWorkerCount(0) + .build(); + queryRunner.createCatalog("healthy_catalog", "memory", ImmutableMap.of("memory.max-data-per-node", "128MB")); + H2QueryRunner h2QueryRunner = new H2QueryRunner(); + + assertQuery(queryRunner, session, "SHOW CATALOGS", h2QueryRunner, "VALUES 'healthy_catalog', '" + BROKEN_CATALOG + "', 'system'", false, false); + assertQueryFails(queryRunner, session, "CREATE TABLE %s.default.test_table (age INT)".formatted(BROKEN_CATALOG), ".*Catalog '%s' failed to initialize and is disabled.*".formatted(BROKEN_CATALOG)); + assertQueryFails(queryRunner, session, "SELECT * FROM %s.default.test_table".formatted(BROKEN_CATALOG), ".*Catalog '%s' failed to initialize and is disabled.*".formatted(BROKEN_CATALOG)); + assertQueryFails(queryRunner, session, "CREATE CATALOG %s USING memory WITH (\"memory.max-data-per-node\" = '128MB')".formatted(BROKEN_CATALOG), ".*Catalog '%s' already exists.*".formatted(BROKEN_CATALOG)); + + assertUpdate(queryRunner, session, "DROP CATALOG " + BROKEN_CATALOG, OptionalLong.empty(), Optional.empty()); + assertQuery(queryRunner, session, "SHOW CATALOGS", h2QueryRunner, "VALUES 'healthy_catalog', 'system'", false, false); + } + + @Test + public void testPrepopulatedHealthyCatalog() + throws Exception + { + Session session = testSession(); + ImmutableMap properties = ImmutableMap.of("memory.max-data-per-node", "128MB"); + QueryRunner queryRunner = DistributedQueryRunner.builder(session) + .setAdditionalModule(new TestCatalogStoreModule(ImmutableMap.of(PREPOPULATED_CATALOG_NAME, new CatalogProperties( + PREPOPULATED_CATALOG_NAME, + new CatalogVersion("abc123"), + MEMORY_CONNECTOR_NAME, + properties)))) + .setAdditionalSetup(runner -> runner.installPlugin(new MemoryPlugin())) + .setCoordinatorProperties(ImmutableMap.of("catalog.store", "prepopulated_memory")) + .setWorkerCount(0) + .build(); + queryRunner.createCatalog("healthy_catalog", "memory", ImmutableMap.of("memory.max-data-per-node", "128MB")); + H2QueryRunner h2QueryRunner = new H2QueryRunner(); + + assertQuery(queryRunner, session, "SHOW CATALOGS", h2QueryRunner, "VALUES 'healthy_catalog', '" + PREPOPULATED_CATALOG + "', 'system'", false, false); + assertUpdate(queryRunner, session, "CREATE TABLE %s.default.test_table (age INT)".formatted(PREPOPULATED_CATALOG), OptionalLong.empty(), Optional.empty()); + assertQueryReturnsEmptyResult(queryRunner, session, "SELECT * FROM %s.default.test_table".formatted(PREPOPULATED_CATALOG)); + assertQueryFails(queryRunner, session, "CREATE CATALOG %s USING memory WITH (\"memory.max-data-per-node\" = '128MB')".formatted(PREPOPULATED_CATALOG), ".*Catalog '%s' already exists.*".formatted(PREPOPULATED_CATALOG)); + + assertUpdate(queryRunner, session, "DROP CATALOG " + PREPOPULATED_CATALOG, OptionalLong.empty(), Optional.empty()); + assertQuery(queryRunner, session, "SHOW CATALOGS", h2QueryRunner, "VALUES 'healthy_catalog', 'system'", false, false); + } + + public static class TestCatalogStoreModule + extends AbstractConfigurationAwareModule + { + private final Map prepopulatedCatalogs; + + public TestCatalogStoreModule(Map prepopulatedCatalogs) + { + this.prepopulatedCatalogs = requireNonNull(prepopulatedCatalogs, "prepopulatedCatalogs is null"); + } + + @Override + protected void setup(Binder binder) + { + if (buildConfigObject(ServerConfig.class).isCoordinator()) { + install(new PrepopulatedInMemoryCatalogStoreModule(prepopulatedCatalogs)); + } + } + } + + private static class PrepopulatedInMemoryCatalogStoreModule + extends AbstractConfigurationAwareModule + { + private final Map prepopulatedCatalogs; + + public PrepopulatedInMemoryCatalogStoreModule(Map prepopulatedCatalogs) + { + this.prepopulatedCatalogs = requireNonNull(prepopulatedCatalogs, "prepopulatedCatalogs is null"); + } + + @Override + protected void setup(Binder binder) {} + + @Provides + @Singleton + public PrepopulatedInMemoryCatalogStoreFactory createDbCatalogStoreFactory(CatalogStoreManager catalogStoreManager) + { + PrepopulatedInMemoryCatalogStoreFactory factory = new PrepopulatedInMemoryCatalogStoreFactory(prepopulatedCatalogs); + catalogStoreManager.addCatalogStoreFactory(factory); + return factory; + } + } + + private static class PrepopulatedInMemoryCatalogStoreFactory + implements CatalogStoreFactory + { + private final Map prepopulatedCatalogs; + + public PrepopulatedInMemoryCatalogStoreFactory(Map prepopulatedCatalogs) + { + this.prepopulatedCatalogs = requireNonNull(prepopulatedCatalogs, "prepopulatedCatalogs is null"); + } + + @Override + public String getName() + { + return "prepopulated_memory"; + } + + @Override + public CatalogStore create(Map config) + { + return new PrepopulatedInMemoryCatalogStore(prepopulatedCatalogs); + } + } + + private static class PrepopulatedInMemoryCatalogStore + extends InMemoryCatalogStore + { + private final Map prepopulatedCatalogs; + + public PrepopulatedInMemoryCatalogStore(Map prepopulatedCatalogs) + { + this.prepopulatedCatalogs = requireNonNull(prepopulatedCatalogs, "prepopulatedCatalogs is null"); + } + + @Override + public Collection getCatalogs() + { + Collection catalogs = super.getCatalogs(); + List catalogsCopy = new ArrayList<>(catalogs); + prepopulatedCatalogs.forEach((catalogName, catalogProperties) -> { + catalogsCopy.add(new StoredCatalog() + { + @Override + public CatalogName name() + { + return catalogName; + } + + @Override + public CatalogProperties loadProperties() + { + return catalogProperties; + } + }); + }); + return catalogsCopy; + } + } +} diff --git a/testing/trino-tests/src/test/java/io/trino/connector/system/TestSystemMetadataCatalogTable.java b/testing/trino-tests/src/test/java/io/trino/connector/system/metadata/TestSystemMetadataCatalogTable.java similarity index 61% rename from testing/trino-tests/src/test/java/io/trino/connector/system/TestSystemMetadataCatalogTable.java rename to testing/trino-tests/src/test/java/io/trino/connector/system/metadata/TestSystemMetadataCatalogTable.java index c990f0bcd575..0e7324b1b70a 100644 --- a/testing/trino-tests/src/test/java/io/trino/connector/system/TestSystemMetadataCatalogTable.java +++ b/testing/trino-tests/src/test/java/io/trino/connector/system/metadata/TestSystemMetadataCatalogTable.java @@ -11,16 +11,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.connector.system; +package io.trino.connector.system.metadata; import com.google.common.collect.ImmutableMap; -import com.google.inject.Key; import io.trino.Session; -import io.trino.connector.ConnectorServicesProvider; +import io.trino.connector.TestDynamicCatalogs; import io.trino.plugin.memory.MemoryPlugin; import io.trino.spi.catalog.CatalogName; import io.trino.spi.catalog.CatalogProperties; -import io.trino.spi.catalog.CatalogStore; +import io.trino.spi.connector.CatalogVersion; import io.trino.spi.connector.ConnectorName; import io.trino.testing.AbstractTestQueryFramework; import io.trino.testing.DistributedQueryRunner; @@ -35,43 +34,52 @@ public class TestSystemMetadataCatalogTable extends AbstractTestQueryFramework { - private CatalogStore catalogStore; - private ConnectorServicesProvider catalogManager; + private static final String BROKEN_CATALOG = "broken_catalog"; @Override protected QueryRunner createQueryRunner() throws Exception { Session session = testSessionBuilder().build(); + ImmutableMap properties = ImmutableMap.of("non_existing", "false"); QueryRunner queryRunner = DistributedQueryRunner.builder(session) + .setAdditionalModule(new TestDynamicCatalogs.TestCatalogStoreModule(ImmutableMap.of(new CatalogName(BROKEN_CATALOG), new CatalogProperties( + new CatalogName(BROKEN_CATALOG), + new CatalogVersion("abc123"), + new ConnectorName("memory"), + properties)))) + .setCoordinatorProperties(ImmutableMap.of("catalog.store", "prepopulated_memory")) .setWorkerCount(0) .build(); queryRunner.installPlugin(new MemoryPlugin()); queryRunner.createCatalog("healthy_catalog", "memory", ImmutableMap.of("memory.max-data-per-node", "128MB")); - catalogStore = queryRunner.getCoordinator().getInstance(new Key<>() {}); - catalogManager = queryRunner.getCoordinator().getInstance(new Key<>() {}); return queryRunner; } @Test - public void testCatalogTableShowsOnlyLoadedCatalogs() + public void testNewCatalogStatus() { assertQuery("SELECT * FROM system.metadata.catalogs", "VALUES" + - "('healthy_catalog', 'healthy_catalog', 'memory'), " + - "('system', 'system', 'system')"); + "('healthy_catalog', 'healthy_catalog', 'memory', 'OPERATIONAL'), " + + "('broken_catalog', 'broken_catalog', 'memory', 'FAILING'), " + + "('system', 'system', 'system', 'OPERATIONAL')"); assertUpdate("CREATE CATALOG brain USING memory WITH (\"memory.max-data-per-node\" = '128MB')"); assertQuery("SELECT * FROM system.metadata.catalogs", "VALUES" + - "('healthy_catalog', 'healthy_catalog', 'memory'), " + - "('brain', 'brain', 'memory'), " + - "('system', 'system', 'system')"); - CatalogProperties catalogProperties = catalogStore.createCatalogProperties(new CatalogName("broken"), new ConnectorName("memory"), ImmutableMap.of("memory.max-data-per-n", "128MB")); - catalogStore.addOrReplaceCatalog(catalogProperties); - catalogManager.loadInitialCatalogs(); + "('healthy_catalog', 'healthy_catalog', 'memory', 'OPERATIONAL'), " + + "('broken_catalog', 'broken_catalog', 'memory', 'FAILING'), " + + "('brain', 'brain', 'memory', 'OPERATIONAL'), " + + "('system', 'system', 'system', 'OPERATIONAL')"); + assertUpdate("DROP CATALOG brain"); + } + + @Test + public void testCatalogNotLoadedCorrectly() + { assertQuery("SELECT * FROM system.metadata.catalogs", "VALUES" + - "('healthy_catalog', 'healthy_catalog', 'memory'), " + - "('brain', 'brain', 'memory'), " + - "('system', 'system', 'system')"); + "('healthy_catalog', 'healthy_catalog', 'memory', 'OPERATIONAL'), " + + "('broken_catalog', 'broken_catalog', 'memory', 'FAILING'), " + + "('system', 'system', 'system', 'OPERATIONAL')"); } } diff --git a/testing/trino-tests/src/test/java/io/trino/connector/system/metadata/TestSystemMetadataConnector.java b/testing/trino-tests/src/test/java/io/trino/connector/system/metadata/TestSystemMetadataConnector.java index a2a0de6fb36f..4a861b2faaa5 100644 --- a/testing/trino-tests/src/test/java/io/trino/connector/system/metadata/TestSystemMetadataConnector.java +++ b/testing/trino-tests/src/test/java/io/trino/connector/system/metadata/TestSystemMetadataConnector.java @@ -407,6 +407,13 @@ public void testMetadataListingExceptionHandling() "Error listing materialized views for catalog broken_catalog: Catalog is broken"); } + @Test + public void testNonExistentTable() + { + assertThat(query("SELECT * FROM system.metadata.non_existent_table")) + .failure().hasMessageContaining("Table 'system.metadata.non_existent_table' does not exist"); + } + private void assertMetadataCalls(@Language("SQL") String actualSql, @Language("SQL") String expectedSql, Multiset expectedMetadataCallsCount) { Multiset actualMetadataCallsCount = countingMockConnector.runTracing(() -> { diff --git a/testing/trino-tests/src/test/java/io/trino/connector/system/runtime/TestSystemRuntimeConnector.java b/testing/trino-tests/src/test/java/io/trino/connector/system/runtime/TestSystemRuntimeConnector.java index e5a386dbacd3..0364bf1715f8 100644 --- a/testing/trino-tests/src/test/java/io/trino/connector/system/runtime/TestSystemRuntimeConnector.java +++ b/testing/trino-tests/src/test/java/io/trino/connector/system/runtime/TestSystemRuntimeConnector.java @@ -299,6 +299,13 @@ public void testTasksTable() getQueryRunner().execute("SELECT * FROM system.runtime.tasks"); } + @Test + public void testNonExistentTable() + { + assertThat(query("SELECT * FROM system.runtime.non_existent_table")) + .failure().hasMessageContaining("Table 'system.runtime.non_existent_table' does not exist"); + } + private static void run(int repetitions, double successRate, Runnable test) { AssertionError lastError = null; diff --git a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java index a6efb40f31d9..6147a1e41719 100644 --- a/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java +++ b/testing/trino-tests/src/test/java/io/trino/execution/TestEventListenerBasic.java @@ -51,6 +51,7 @@ import io.trino.spi.eventlistener.TableInfo; import io.trino.spi.metrics.Metrics; import io.trino.spi.security.ViewExpression; +import io.trino.spi.type.ArrayType; import io.trino.spi.type.Type; import io.trino.spi.type.TypeManager; import io.trino.spi.type.TypeSignature; @@ -156,7 +157,9 @@ public Iterable getConnectorFactories() } return ImmutableList.of( new ColumnMetadata("test_varchar", createVarcharType(15)), - new ColumnMetadata("test_bigint", BIGINT)); + new ColumnMetadata("test_bigint", BIGINT), + new ColumnMetadata("test_varchar_array", new ArrayType(createVarcharType(15))), + new ColumnMetadata("test_bigint_array", new ArrayType(BIGINT))); }) .withGetTableHandle((session, schemaTableName) -> { if (!schemaTableName.getTableName().startsWith("create")) { @@ -761,7 +764,7 @@ public void testReferencedTablesWithColumnMask() throws Exception { QueryEvents queryEvents = runQueryAndWaitForEvents( - "CREATE TABLE mock.default.create_table_with_referring_mask AS SELECT * FROM mock.default.test_table_with_column_mask" + "CREATE TABLE mock.default.create_table_with_referring_mask AS SELECT test_varchar, test_bigint FROM mock.default.test_table_with_column_mask" ).getQueryEvents(); QueryCompletedEvent event = queryEvents.getQueryCompletedEvent(); @@ -1372,7 +1375,9 @@ public void testCreateTableLike() .containsExactly( new OutputColumnMetadata("test_column", BIGINT_TYPE, ImmutableSet.of()), new OutputColumnMetadata("test_varchar", VARCHAR_TYPE, ImmutableSet.of()), - new OutputColumnMetadata("test_bigint", BIGINT_TYPE, ImmutableSet.of())); + new OutputColumnMetadata("test_bigint", BIGINT_TYPE, ImmutableSet.of()), + new OutputColumnMetadata("test_varchar_array", "array(varchar(15))", ImmutableSet.of()), + new OutputColumnMetadata("test_bigint_array", "array(bigint)", ImmutableSet.of())); } @Test @@ -1419,6 +1424,41 @@ private void testOutputColumnsForSetOperations(String setOperator) new ColumnDetail("tpch", "sf1", "orders", "custkey")))); } + @Test + public void testOutputColumnsWithUnnestQueries() + throws Exception + { + // test unnest with one array column + assertLineage( + "SELECT test_varchar_unnest AS test_varchar, test_bigint AS test_bigint FROM mock.default.tests_table_unnest CROSS JOIN UNNEST(test_varchar_array) AS t(test_varchar_unnest)", + ImmutableSet.of("mock.default.tests_table_unnest"), + new OutputColumnMetadata("test_varchar", VARCHAR_TYPE, ImmutableSet.of(new ColumnDetail("mock", "default", "tests_table_unnest", "test_varchar_array"))), + new OutputColumnMetadata("test_bigint", BIGINT_TYPE, ImmutableSet.of(new ColumnDetail("mock", "default", "tests_table_unnest", "test_bigint")))); + // test unnest with one array column and with ordinality + assertLineage( + "SELECT test_varchar_unnest AS test_varchar, row_number_unnest AS test_bigint FROM mock.default.tests_table_unnest CROSS JOIN UNNEST(test_varchar_array) WITH ORDINALITY AS t(test_varchar_unnest, row_number_unnest)", + ImmutableSet.of("mock.default.tests_table_unnest"), + new OutputColumnMetadata("test_varchar", VARCHAR_TYPE, ImmutableSet.of(new ColumnDetail("mock", "default", "tests_table_unnest", "test_varchar_array"))), + new OutputColumnMetadata("test_bigint", BIGINT_TYPE, ImmutableSet.of())); + // test unnest with two array column + assertLineage( + "SELECT test_varchar_unnest AS test_varchar, test_bigint_unnest AS test_bigint FROM mock.default.tests_table_unnest CROSS JOIN UNNEST(test_varchar_array, test_bigint_array) AS t(test_varchar_unnest, test_bigint_unnest)", + ImmutableSet.of("mock.default.tests_table_unnest"), + new OutputColumnMetadata("test_varchar", VARCHAR_TYPE, ImmutableSet.of(new ColumnDetail("mock", "default", "tests_table_unnest", "test_varchar_array"))), + new OutputColumnMetadata("test_bigint", BIGINT_TYPE, ImmutableSet.of(new ColumnDetail("mock", "default", "tests_table_unnest", "test_bigint_array")))); + // test unnest with two array column and with ordinality + assertLineage( + "SELECT test_varchar_unnest AS test_varchar, row_number_unnest AS test_bigint FROM mock.default.tests_table_unnest CROSS JOIN UNNEST(test_varchar_array, test_bigint_array) WITH ORDINALITY AS t(test_varchar_unnest, test_bigint_unnest, row_number_unnest)", + ImmutableSet.of("mock.default.tests_table_unnest"), + new OutputColumnMetadata("test_varchar", VARCHAR_TYPE, ImmutableSet.of(new ColumnDetail("mock", "default", "tests_table_unnest", "test_varchar_array"))), + new OutputColumnMetadata("test_bigint", BIGINT_TYPE, ImmutableSet.of())); + assertLineage( + "SELECT CAST(test_bigint_unnest AS varchar(15)) AS test_varchar, row_number_unnest AS test_bigint FROM mock.default.tests_table_unnest CROSS JOIN UNNEST(test_varchar_array, test_bigint_array) WITH ORDINALITY AS t(test_varchar_unnest, test_bigint_unnest, row_number_unnest)", + ImmutableSet.of("mock.default.tests_table_unnest"), + new OutputColumnMetadata("test_varchar", VARCHAR_TYPE, ImmutableSet.of(new ColumnDetail("mock", "default", "tests_table_unnest", "test_bigint_array"))), + new OutputColumnMetadata("test_bigint", BIGINT_TYPE, ImmutableSet.of())); + } + @Test public void testTableStats() throws Exception diff --git a/testing/trino-tests/src/test/java/io/trino/execution/TestQueryCompletedEvent.java b/testing/trino-tests/src/test/java/io/trino/execution/TestQueryCompletedEvent.java index 8d2ea9f5e537..466f13f6a691 100644 --- a/testing/trino-tests/src/test/java/io/trino/execution/TestQueryCompletedEvent.java +++ b/testing/trino-tests/src/test/java/io/trino/execution/TestQueryCompletedEvent.java @@ -86,6 +86,6 @@ private static void assertQueryCompletedIssued(StandaloneQueryRunner queryRunner assertThat(query.getState()).isEqualTo(FINISHED); assertEventually(() -> assertThat(queryCompletedQueryIds) .describedAs("query '%s'", sql) - .contains(result.queryId().getId())); + .contains(result.queryId().id())); } } diff --git a/testing/trino-tests/src/test/java/io/trino/execution/resourcegroups/db/H2ResourceGroupConfigurationManagerFactory.java b/testing/trino-tests/src/test/java/io/trino/execution/resourcegroups/db/H2ResourceGroupConfigurationManagerFactory.java index 7100862ab86f..c53fc1141ac7 100644 --- a/testing/trino-tests/src/test/java/io/trino/execution/resourcegroups/db/H2ResourceGroupConfigurationManagerFactory.java +++ b/testing/trino-tests/src/test/java/io/trino/execution/resourcegroups/db/H2ResourceGroupConfigurationManagerFactory.java @@ -58,6 +58,7 @@ public ResourceGroupConfigurationManager create(Map config, R Injector injector = app .doNotInitializeLogging() + .disableSystemProperties() .setRequiredConfigurationProperties(config) .quiet() .initialize(); diff --git a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java index f0078edcb9a4..441ae7740df5 100644 --- a/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java +++ b/testing/trino-tests/src/test/java/io/trino/memory/TestClusterMemoryLeakDetector.java @@ -50,15 +50,15 @@ public void testLeakDetector() assertThat(leakDetector.getNumberOfLeakedQueries()).isEqualTo(0); // the leak detector should report no leaked queries as the query is still running - leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), RUNNING)), ImmutableMap.of(testQuery, 1L)); + leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.id(), RUNNING)), ImmutableMap.of(testQuery, 1L)); assertThat(leakDetector.getNumberOfLeakedQueries()).isEqualTo(0); // the leak detector should report exactly one leaked query since the query is finished, and its end time is way in the past - leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 1L)); + leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.id(), FINISHED)), ImmutableMap.of(testQuery, 1L)); assertThat(leakDetector.getNumberOfLeakedQueries()).isEqualTo(1); // the leak detector should report no leaked queries as the query doesn't have any memory reservation - leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.getId(), FINISHED)), ImmutableMap.of(testQuery, 0L)); + leakDetector.checkForMemoryLeaks(() -> ImmutableList.of(createQueryInfo(testQuery.id(), FINISHED)), ImmutableMap.of(testQuery, 0L)); assertThat(leakDetector.getNumberOfLeakedQueries()).isEqualTo(0); // the leak detector should report exactly one leaked query since the coordinator doesn't know of any query diff --git a/testing/trino-tests/src/test/java/io/trino/security/TestingSystemSecurityMetadata.java b/testing/trino-tests/src/test/java/io/trino/security/TestingSystemSecurityMetadata.java index 12d5eea11439..83c664d66f65 100644 --- a/testing/trino-tests/src/test/java/io/trino/security/TestingSystemSecurityMetadata.java +++ b/testing/trino-tests/src/test/java/io/trino/security/TestingSystemSecurityMetadata.java @@ -20,6 +20,7 @@ import io.trino.metadata.QualifiedSchemaPrefix; import io.trino.metadata.QualifiedTablePrefix; import io.trino.metadata.SystemSecurityMetadata; +import io.trino.spi.catalog.CatalogName; import io.trino.spi.connector.CatalogSchemaName; import io.trino.spi.connector.CatalogSchemaTableName; import io.trino.spi.connector.EntityKindAndName; @@ -272,6 +273,12 @@ public Optional getFunctionRunAsIdentity(Session session, CatalogSchem .build()); } + @Override + public void catalogCreated(Session session, CatalogName catalog) {} + + @Override + public void catalogDropped(Session session, CatalogName catalog) {} + @Override public void functionCreated(Session session, CatalogSchemaFunctionName function) { diff --git a/testing/trino-tests/src/test/java/io/trino/sql/planner/BaseCostBasedPlanTest.java b/testing/trino-tests/src/test/java/io/trino/sql/planner/BaseCostBasedPlanTest.java index 68206b5a744b..4cf6ad7a1b93 100644 --- a/testing/trino-tests/src/test/java/io/trino/sql/planner/BaseCostBasedPlanTest.java +++ b/testing/trino-tests/src/test/java/io/trino/sql/planner/BaseCostBasedPlanTest.java @@ -44,7 +44,9 @@ import java.io.UncheckedIOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.google.common.base.Preconditions.checkState; @@ -148,6 +150,29 @@ public void test(String queryResourcePath) assertThat(generateQueryPlan(readQuery(queryResourcePath))).isEqualTo(read(getQueryPlanResourcePath(queryResourcePath))); } + protected void assertExplainAnalyzePlan(String queryResourcePath) + { + String query = readQuery(queryResourcePath); + + String queryPlan = generateQueryPlan(query); + String explainAnalyzeQueryPlan = generateQueryPlan("EXPLAIN ANALYZE " + query); + String[] explainAnalyzeLines = explainAnalyzeQueryPlan.split("\n"); + + // for EXPLAIN ANALYZE, the first two lines reflect the additional root fragment containing the ExplainAnalyze operator + assertThat(String.join("\n", Arrays.copyOfRange(explainAnalyzeLines, 0, 2)) + "\n") + .isEqualTo(""" + local exchange (GATHER, SINGLE, []) + remote exchange (GATHER, SINGLE, []) + """); + + // the remaining lines should match the original query plan, except for the indentation + explainAnalyzeQueryPlan = Arrays.stream(Arrays.copyOfRange(explainAnalyzeLines, 2, explainAnalyzeLines.length)) + .map(line -> line.replaceFirst("^ {8}", "")) + .collect(Collectors.joining("\n")) + "\n"; + + assertThat(queryPlan).isEqualTo(explainAnalyzeQueryPlan); + } + private String getQueryPlanResourcePath(String queryResourcePath) { Path queryPath = Paths.get(queryResourcePath); diff --git a/testing/trino-tests/src/test/java/io/trino/sql/planner/TestExplainAnalyzePartitionedTpcdsPlan.java b/testing/trino-tests/src/test/java/io/trino/sql/planner/TestExplainAnalyzePartitionedTpcdsPlan.java new file mode 100644 index 000000000000..6fe1d3b3c6dd --- /dev/null +++ b/testing/trino-tests/src/test/java/io/trino/sql/planner/TestExplainAnalyzePartitionedTpcdsPlan.java @@ -0,0 +1,66 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import io.trino.tpcds.Table; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +public class TestExplainAnalyzePartitionedTpcdsPlan + extends BaseCostBasedPlanTest +{ + protected TestExplainAnalyzePartitionedTpcdsPlan() + { + super("tpcds_sf1000_parquet_part", true); + } + + @Override + @ParameterizedTest + @MethodSource("getQueryResourcePaths") + public void test(String queryResourcePath) + { + assertExplainAnalyzePlan(queryResourcePath); + } + + @Override + protected List getTableNames() + { + return io.trino.tpcds.Table.getBaseTables().stream() + .filter(table -> table != io.trino.tpcds.Table.DBGEN_VERSION) + .map(Table::getName) + .collect(toImmutableList()); + } + + @Override + protected String getTableResourceDirectory() + { + return "iceberg/tpcds/sf1000/partitioned/"; + } + + @Override + protected String getTableTargetDirectory() + { + return "iceberg-tpcds-sf1000-parquet-part/"; + } + + @Override + protected List getQueryResourcePaths() + { + return TPCDS_SQL_FILES; + } +} diff --git a/testing/trino-tests/src/test/java/io/trino/sql/planner/TestExplainAnalyzeTpchPlan.java b/testing/trino-tests/src/test/java/io/trino/sql/planner/TestExplainAnalyzeTpchPlan.java new file mode 100644 index 000000000000..b72c256dc2f0 --- /dev/null +++ b/testing/trino-tests/src/test/java/io/trino/sql/planner/TestExplainAnalyzeTpchPlan.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import io.trino.tpch.TpchTable; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +public class TestExplainAnalyzeTpchPlan + extends BaseCostBasedPlanTest +{ + public TestExplainAnalyzeTpchPlan() + { + super("tpch_sf1000_parquet", false); + } + + @Override + @ParameterizedTest + @MethodSource("getQueryResourcePaths") + public void test(String queryResourcePath) + { + assertExplainAnalyzePlan(queryResourcePath); + } + + @Override + protected List getTableNames() + { + return TpchTable.getTables().stream() + .map(TpchTable::getTableName) + .collect(toImmutableList()); + } + + @Override + protected String getTableResourceDirectory() + { + return "iceberg/tpch/sf1000/unpartitioned/"; + } + + @Override + protected String getTableTargetDirectory() + { + return "iceberg-tpch-sf1000-parquet/"; + } + + @Override + protected List getQueryResourcePaths() + { + return TPCH_SQL_FILES; + } +} diff --git a/testing/trino-tests/src/test/java/io/trino/tests/ci/TestCiWorkflow.java b/testing/trino-tests/src/test/java/io/trino/tests/ci/TestCiWorkflow.java index de5a22caf168..d2cf51249207 100644 --- a/testing/trino-tests/src/test/java/io/trino/tests/ci/TestCiWorkflow.java +++ b/testing/trino-tests/src/test/java/io/trino/tests/ci/TestCiWorkflow.java @@ -33,8 +33,10 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.Iterables.getLast; import static com.google.common.collect.MoreCollectors.onlyElement; import static java.util.Locale.ENGLISH; +import static java.util.function.Predicate.not; import static org.assertj.core.api.Assertions.assertThat; public class TestCiWorkflow @@ -42,6 +44,7 @@ public class TestCiWorkflow private static final Logger log = Logger.get(TestCiWorkflow.class); private static final Path CI_YML_REPO_PATH = Paths.get(".github/workflows/ci.yml"); + private static final String BUILD_SUCCESS = "build-success"; @Test public void testUploadTestResultsCondition() @@ -96,6 +99,57 @@ public void testUploadTestResultsCondition() .containsExactlyInAnyOrderElementsOf(testStepIds); } + @Test + public void testBuildSuccessDependencies() + throws Exception + { + Yaml yaml = new Yaml(); + Map workflow = yaml.load(new StringReader(Files.readString(findRepositoryRoot().resolve(CI_YML_REPO_PATH)))); + Map jobs = getMap(workflow, "jobs"); + + Set allJobNames = jobs.keySet(); + assertThat(allJobNames).as("allJobNames").contains(BUILD_SUCCESS); + List testJobNames = allJobNames.stream() + .filter(not(BUILD_SUCCESS::equals)) + .sorted() + .toList(); + Map buildSuccessJob = getMap(jobs, BUILD_SUCCESS); + + List buildSuccessDependencies = getList(buildSuccessJob, "needs").stream() + .map(String.class::cast) + .toList(); + assertThat(buildSuccessDependencies).as("dependencies for %s", BUILD_SUCCESS) + .isSorted() + .doesNotHaveDuplicates() + .containsExactlyElementsOf(testJobNames); + + StringBuilder expectedRunDefinition = new StringBuilder(); + expectedRunDefinition.append("# generated by %s\n".formatted(getClass().getSimpleName())); + for (String name : testJobNames) { + expectedRunDefinition.append( + "echo '${{ needs.%1$s.result }}' | grep -xE 'success|skipped' || { echo 'Job \"%1$s\" failed' >&2; exit 1; }\n".formatted(name)); + } + Map runDefinition = getList(buildSuccessJob, "steps").stream() + .map(step -> (Map) step) + .filter(step -> "Check results".equals(step.get("name"))) + .collect(onlyElement()); + assertThat(runDefinition.get("run")).as("run script") + .isEqualTo(expectedRunDefinition.toString()); + } + + @Test + public void testBuildSuccessIsLast() + throws Exception + { + Yaml yaml = new Yaml(); + Map workflow = yaml.load(new StringReader(Files.readString(findRepositoryRoot().resolve(CI_YML_REPO_PATH)))); + Map jobs = getMap(workflow, "jobs"); + // This assumes the `jobs` map preserves source order + assertThat(getLast(jobs.keySet())) + .describedAs("The %s job is logically last and depends on all others, we want it to be last in the source file", BUILD_SUCCESS) + .isEqualTo(BUILD_SUCCESS); + } + private static Path findRepositoryRoot() { Path workingDirectory = Paths.get("").toAbsolutePath(); diff --git a/testing/trino-tests/src/test/java/io/trino/tests/tpch/TestTpchConnectorTest.java b/testing/trino-tests/src/test/java/io/trino/tests/tpch/TestTpchConnectorTest.java index 360bc8ef1f4f..c3d64c24607c 100644 --- a/testing/trino-tests/src/test/java/io/trino/tests/tpch/TestTpchConnectorTest.java +++ b/testing/trino-tests/src/test/java/io/trino/tests/tpch/TestTpchConnectorTest.java @@ -62,6 +62,7 @@ protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) SUPPORTS_CREATE_VIEW, SUPPORTS_DELETE, SUPPORTS_INSERT, + SUPPORTS_LIMIT_PUSHDOWN, SUPPORTS_MAP_TYPE, SUPPORTS_MERGE, SUPPORTS_RENAME_COLUMN,