Merge branch 'main' into feat_fix

Author: zhyass

.github/actions/setup_docker/action.yml

@@ -20,6 +20,9 @@ outputs:
   ecr_repo:
     description: "AWS public ECR Repository"
     value: ${{ steps.registry.outputs.ecr }}
+  ghcr_repo:
+    description: "GitHub Container Registry Repository"
+    value: ${{ steps.registry.outputs.ghcr }}
 
 runs:
   using: "composite"
@@ -57,9 +60,17 @@ runs:
         username: ${{ inputs.dockerhub_user }}
         password: ${{ inputs.dockerhub_token }}
 
+    - name: Login to GHCR
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ github.token }}
+
     - name: Output Registries
       shell: bash
       id: registry
       run: |
         echo 'dockerhub=datafuselabs/${{ inputs.repo }}' >> $GITHUB_OUTPUT
         echo 'ecr=public.ecr.aws/i7g1w5q7/${{ inputs.repo }}' >> $GITHUB_OUTPUT
+        echo 'ghcr=ghcr.io/${{ github.repository_owner }}/${{ inputs.repo }}' >> $GITHUB_OUTPUT

.github/actions/test_sqllogic_iceberg_tpch/action.yml

@@ -31,6 +31,7 @@ runs:
 
         pip install pyspark
         python3 tests/sqllogictests/scripts/prepare_iceberg_tpch_data.py
+        python3 tests/sqllogictests/scripts/prepare_iceberg_test_data.py
 
 
     - name: Run sqllogic Tests with Standalone lib

.github/actions/test_stateful_iceberg_catalogs_standalone/action.yml

@@ -0,0 +1,21 @@
+name: "Test Stateful Standalone Iceberg Rest"
+description: "Running Stateful Iceberg Rest tests in standalone mode"
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/actions/setup_test
+
+    - name: Iceberg Rest Setup for (ubuntu-latest only)
+      shell: bash
+      run: |
+        DOCKER_DEFAULT_PLATFORM=`docker info --format {{.OSType}}/{{.Architecture}}`
+        docker-compose -f "./docker/it-iceberg-catalogs/docker-compose.yaml" up --quiet-pull -d
+        hms_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -aq --filter "name=hive-metastore"))
+
+        docker ps -a | grep iceberg
+        echo "HMS in $hms_ip"
+
+    - name: Run Stateful Tests with Standalone mode
+      shell: bash
+      run: |
+        ./scripts/ci/ci-run-stateful-iceberg-catalogs-tests-standalone.sh

.github/actions/test_stateful_iceberg_rest_standalone/action.yml

@@ -2,15 +2,15 @@
 
 ## Release
 
-Databend make new releases via workflow [Release](./workflows/release.yml).
+Databend makes new releases via workflow [Release](./workflows/release.yml).
 
 There are three ways to trigger a new release:
 
 - Push a new tag
 - On Schedule (every day at 00:00 UTC+8)
 - Manually triggered
 
-If new tags pushed, we use this the new tag name as release name. Otherwise, we will generate a new nightly tag.
+If new tags pushed, we use the new tag name as release name. Otherwise, we will generate a new nightly tag.
 
 For example:
 
@@ -23,4 +23,4 @@ For every release, we will:
 - Build binaries and pack them on Linux and MacOS.
 - Upload built packages to GitHub Releases, AWS S3 bucket and Docker Registry.
 
-We are adopting github native release notes generation which controlled by [release.yml](./release.yml).
+We are adopting github native release notes generation which is controlled by [release.yml](./release.yml).

.github/workflows.md

@@ -11,6 +11,7 @@ permissions:
   id-token: write
   pull-requests: write
   contents: read
+  packages: write
 
 env:
   BUILD_PROFILE: release
@@ -97,17 +98,35 @@ jobs:
         with:
           repo: databend-query
           ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }}
+          dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Prepare for docker
         id: prepare
         run: |
           short_sha=$(echo "${{ needs.info.outputs.sha }}" | cut -c1-7)
           ts=$(date +%s)
           echo "tag=pr-${{ github.event.pull_request.number }}-${short_sha}-${ts}" >> $GITHUB_OUTPUT
+      - name: Get Image Tags
+        id: tags
+        uses: actions/github-script@v7
+        env:
+          REPO_DOCKERHUB: ${{ steps.login.outputs.dockerhub_repo }}
+          REPO_ECR: ${{ steps.login.outputs.ecr_repo }}
+          VERSION: ${{ steps.prepare.outputs.tag }}
+        with:
+          script: |
+            const { VERSION, REPO_DOCKERHUB, REPO_ECR } = process.env;
+            const repos = [REPO_DOCKERHUB, REPO_ECR];
+            let tags = [];
+            for (const repo of repos) {
+              tags.push(`${repo}:${VERSION}`);
+            }
+            core.setOutput('tags', tags.join(','));
       - name: push service image
         uses: docker/build-push-action@v5
         with:
           push: true
-          tags: ${{ steps.login.outputs.ecr_repo }}:${{ steps.prepare.outputs.tag }}
+          tags: ${{ steps.tags.outputs.tags }}
           platforms: linux/amd64,linux/arm64
           context: .
           file: ./docker/service/query.Dockerfile

.github/workflows/cloud.yml

@@ -29,6 +29,7 @@ permissions:
   checks: write
   statuses: write
   contents: write
+  packages: write
 
 env:
   BUILD_PROFILE: release

.github/workflows/release.yml

@@ -211,18 +211,18 @@ jobs:
       - uses: ./.github/actions/test_stateful_large_data
         timeout-minutes: 60
 
-  test_stateful_iceberg_rest:
+  test_stateful_iceberg_catalogs:
     runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
     needs: [build, check]
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/test_stateful_iceberg_rest_standalone
+      - uses: ./.github/actions/test_stateful_iceberg_catalogs_standalone
         timeout-minutes: 10
       - name: Upload failure
         if: failure()
         uses: ./.github/actions/artifact_failure
         with:
-          name: test-stateful-iceberg-rest-standalone
+          name: test-stateful-iceberg-catalogs-standalone
 
   test_stateful_hive_standalone:
     needs: [build, check]
@@ -235,7 +235,7 @@ jobs:
         if: failure()
         uses: ./.github/actions/artifact_failure
         with:
-          name: test-stateful-iceberg-rest-standalone
+          name: test_stateful_hive_standalone
 
   # test_fuzz_standalone:
   #   runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]