Merge branch 'main' into orca-support-two-step-window-func

Author: yjhjstz

.asf.yaml

@@ -60,6 +60,8 @@ github:
     issues: true
     # Enable GitHub Projects for project management and planning
     projects: true
+    # Enable GitHub Discussions for community discussions
+    discussions: true
 
   # Configure available merge strategies for pull requests
   enabled_merge_buttons:
@@ -88,6 +90,10 @@ github:
           - RPM Install Test Apache Cloudberry
           - ic-good-opt-off
           - ic-good-opt-on
+          - pax-ic-good-opt-off
+          - pax-ic-good-opt-on
+          - pax-ic-isolation2-opt-off
+          - pax-ic-isolation2-opt-on
           - ic-expandshrink
           - ic-singlenode
           - ic-resgroup-v2

.github/workflows/build-cloudberry.yml

@@ -230,6 +230,42 @@ jobs:
                "make_configs":["src/test/regress:installcheck-good"],
                "pg_settings":{"optimizer":"on"}
               },
+              {"test":"pax-ic-good-opt-off",
+               "make_configs":[
+                "contrib/pax_storage/:pax-test",
+                "contrib/pax_storage/:regress_test"
+              ],
+               "pg_settings":{
+                 "optimizer":"off",
+                 "default_table_access_method":"pax"
+                }
+              },
+              {"test":"pax-ic-good-opt-on",
+               "make_configs":[
+                "contrib/pax_storage/:pax-test",
+                "contrib/pax_storage/:regress_test"
+              ],
+               "pg_settings":{
+                 "optimizer":"on",
+                 "default_table_access_method":"pax"
+                }
+              },
+              {"test":"pax-ic-isolation2-opt-off",
+               "make_configs":["contrib/pax_storage/:isolation2_test"],
+               "pg_settings":{
+                 "optimizer":"off",
+                 "default_table_access_method":"pax"
+                },
+                "enable_core_check":false
+              },
+              {"test":"pax-ic-isolation2-opt-on",
+               "make_configs":["contrib/pax_storage/:isolation2_test"],
+               "pg_settings":{
+                 "optimizer":"on",
+                 "default_table_access_method":"pax"
+                },
+                "enable_core_check":false
+              },
               {"test":"ic-expandshrink",
                "make_configs":["src/test/isolation2:installcheck-expandshrink"]
               },
@@ -482,6 +518,11 @@ jobs:
         run: |
           set -eo pipefail
 
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && git submodule update --init"; then
+            echo "::error::Fail to pull submodule."
+            exit 1
+          fi
+
           chmod +x "${SRC_DIR}"/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh
           if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh"; then
             echo "::error::Build script failed"
@@ -1267,6 +1308,10 @@ jobs:
             PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}"
           fi
 
+          if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then
+            PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}"
+          fi
+
           # Read configs into array
           IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}"
 

.github/workflows/build-dbg-cloudberry.yml

@@ -427,6 +427,11 @@ jobs:
         run: |
           set -eo pipefail
 
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && git submodule update --init"; then
+            echo "::error::Fail to pull submodule."
+            exit 1
+          fi
+
           chmod +x "${SRC_DIR}"/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh
           if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh"; then
             echo "::error::Build script failed"

.github/workflows/coverity.yml

@@ -40,7 +40,7 @@ name: Apache Cloudberry Coverity Scan
 
 on:
   schedule:
-  - cron: "0 0 * * *"
+  - cron: "0 0 * * 1"
   workflow_dispatch:
 
 permissions:
@@ -62,7 +62,8 @@ jobs:
       uses: actions/checkout@v4
       with:
         fetch-depth: 1
-       
+        submodules: true
+
     - name: Environment Initialization
       run: |
         if ! su - gpadmin -c "/tmp/init_system.sh"; then
@@ -72,15 +73,16 @@ jobs:
 
     - name: Download Coverity build tool
       run: |
+        set -euox pipefail
         wget -c -N https://scan.coverity.com/download/linux64 --post-data "token=${{ secrets.COVERITY_SCAN_TOKEN }}&project=apache%2Fcloudberry" -O coverity_tool.tar.gz
         mkdir -p coverity_tool
         tar xzf coverity_tool.tar.gz --strip 1 -C coverity_tool
         chown -R gpadmin:gpadmin coverity_tool
 
     - name: Build with Coverity build tool
       run: |
+        set -euox pipefail
         WORKSPACE="${GITHUB_WORKSPACE}"
-        sudo rm -rf /usr/local/cloudberry-db
         sudo chmod a+w /usr/local
         mkdir -p /usr/local/cloudberry-db/lib
         sudo cp /usr/local/xerces-c/lib/libxerces-c.so \
@@ -96,6 +98,7 @@ jobs:
             --enable-ic-proxy \
             --enable-orafce \
             --enable-orca \
+            --enable-pax \
             --enable-pxf \
             --enable-tap-tests \
             --with-gssapi \
@@ -122,4 +125,4 @@ jobs:
           --form [email protected] \
           --form version="Commit $GITHUB_SHA" \
           --form description="Build submitted via CI" \
-          https://scan.coverity.com/builds?project=apache%2Fcloudberry
+          https://scan.coverity.com/builds?project=apache%2Fcloudberry

.github/workflows/sonarqube.yml

@@ -0,0 +1,120 @@
+# --------------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed
+# with this work for additional information regarding copyright
+# ownership. The ASF licenses this file to You under the Apache
+# License, Version 2.0 (the "License"); you may not use this file
+# except in compliance with the License. You may obtain a copy of the
+# License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# --------------------------------------------------------------------
+# GitHub Actions Workflow: Apache Cloudberry SonarQube Pipeline
+# --------------------------------------------------------------------
+# Description:
+#
+#   This workflow performs scheduled SonarQube analysis for Cloudberry.
+#
+# Workflow Overview:
+# 1. **Check Skip**:
+#    - workflow run currently is limited to "apache" GitHub organization
+#
+# 2. **scan Job**:
+#    - performs scan and upload result to https://sonarcloud.io/project/overview?id=apache_cloudberry
+# Triggers:
+# - Weekly schedule
+# - optional manual dispatch.
+#
+# Notes:
+# - SONARCLOUD_TOKEN secret is provided by the ASF Infra team
+# --------------------------------------------------------------------
+name: Apache Cloudberry SonarQube Cloud Analysis
+
+on:
+  schedule:
+  - cron: "0 0 * * 1"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  sonarqube-analysis:
+    runs-on: ubuntu-22.04
+    if: ${{ github.repository_owner == 'apache' }}
+
+    container:
+      image: apache/incubator-cloudberry:cbdb-build-rocky9-latest
+      options: >-
+        --user root
+        -h cdw
+    env:
+      BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
+
+    steps:
+    - name: Checkout Apache Cloudberry
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
+        submodules: true
+
+    - name: Environment Initialization
+      run: |
+        if ! su - gpadmin -c "/tmp/init_system.sh"; then
+          echo "::error::Container initialization failed"
+          exit 1
+        fi
+
+    - name: Install Build Wrapper
+      uses: SonarSource/sonarqube-scan-action/install-build-wrapper@v5
+
+    - name: Run Build Wrapper
+      run: |
+        set -euox pipefail
+        sudo chmod a+w /usr/local
+        mkdir -p /usr/local/cloudberry-db/lib
+        sudo cp /usr/local/xerces-c/lib/libxerces-c.so \
+             /usr/local/xerces-c/lib/libxerces-c-3.3.so \
+             /usr/local/cloudberry-db/lib
+        sudo chown -R gpadmin:gpadmin /usr/local/cloudberry-db
+        export LD_LIBRARY_PATH=/usr/local/cloudberry-db/lib:LD_LIBRARY_PATH
+        ./configure --prefix=/usr/local/cloudberry-db \
+            --disable-external-fts \
+            --enable-gpcloud \
+            --enable-ic-proxy \
+            --enable-orafce \
+            --enable-orca \
+            --enable-pax \
+            --enable-pxf \
+            --enable-tap-tests \
+            --with-gssapi \
+            --with-ldap \
+            --with-libxml \
+            --with-lz4 \
+            --with-openssl \
+            --with-pam \
+            --with-perl \
+            --with-pgport=5432 \
+            --with-python \
+            --with-pythonsrc-ext \
+            --with-ssl=openssl \
+            --with-uuid=e2fs \
+            --with-includes=/usr/local/xerces-c/include \
+            --with-libraries=/usr/local/cloudberry-db/lib
+        build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} make -j$(nproc)
+
+    - name: SonarQube Scan
+      uses: SonarSource/sonarqube-scan-action@v5
+      env:
+        SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}
+      with:
+        args: >
+          --define sonar.cfamily.compile-commands="${{ env.BUILD_WRAPPER_OUT_DIR }}/compile_commands.json"

.gitmodules

@@ -1,3 +1,20 @@
 [submodule "gpcontrib/gpcloud/test/googletest"]
 	path = gpcontrib/gpcloud/test/googletest
 	url = https://github.com/google/googletest.git
+[submodule "contrib/pax_storage/src/cpp/contrib/googletest"]
+	path = contrib/pax_storage/src/cpp/contrib/googletest
+	url = https://github.com/google/googletest.git
+	branch = v1.15.x
+[submodule "contrib/pax_storage/src/cpp/contrib/tabulate"]
+	path = contrib/pax_storage/src/cpp/contrib/tabulate
+	url = https://github.com/p-ranav/tabulate.git
+[submodule "contrib/pax_storage/src/cpp/contrib/googlebench"]
+	path = contrib/pax_storage/src/cpp/contrib/googlebench
+	url = https://github.com/google/benchmark.git
+[submodule "contrib/pax_storage/src/cpp/contrib/cpp-stub"]
+	path = contrib/pax_storage/src/cpp/contrib/cpp-stub
+	url = https://github.com/coolxv/cpp-stub.git
+[submodule "dependency/yyjson"]
+	path = dependency/yyjson
+	url = https://github.com/ibireme/yyjson.git
+

GNUmakefile.in

@@ -31,6 +31,9 @@ all:
 	$(MAKE) -C contrib/pg_buffercache all
 ifeq ($(with_openssl), yes)
 	$(MAKE) -C contrib/sslinfo all
+endif
+ifeq ($(enable_pax), yes)
+	$(MAKE) -C contrib/pax_storage all
 endif
 	$(MAKE) -C gpMgmt all
 	$(MAKE) -C gpcontrib all
@@ -73,6 +76,9 @@ install:
 	$(MAKE) -C contrib/tablefunc $@
 	$(MAKE) -C contrib/passwordcheck $@
 	$(MAKE) -C contrib/pg_buffercache $@
+ifeq ($(enable_pax), yes)
+	$(MAKE) -C contrib/pax_storage $@
+endif
 ifeq ($(with_openssl), yes)
 	$(MAKE) -C contrib/sslinfo $@
 endif
@@ -180,6 +186,9 @@ ICW_TARGETS += contrib/extprotocol contrib/dblink contrib/pg_trgm
 ICW_TARGETS += contrib/indexscan contrib/hstore contrib/pgcrypto
 ICW_TARGETS += contrib/tablefunc contrib/passwordcheck
 ICW_TARGETS += contrib/pg_buffercache
+ifeq ($(enable_pax), yes)
+ICW_TARGETS += contrib/pax_storage
+endif
 # sslinfo depends on openssl
 ifeq ($(with_openssl), yes)
 ICW_TARGETS += contrib/sslinfo

LICENSE

@@ -213,6 +213,12 @@ The PostgreSQL software includes:
       src/bin/pg_controldata/pg_controldata.c
       copyright (c) Oliver Elphick <[email protected]>, 2001
 
+      gpMgmt/bin/pythonSrc/ext/behave-1.2.6.tar.gz
+      see licenses/LICENSE-bsd2-behave.txt
+
+      gpMgmt/bin/pythonSrc/ext/mock-1.0.1.tar.gz
+      see licenses/LICENSE-bsd2-mock.txt
+
 ----------------------------
    BSD-4-Clause (University of California-Specific)
 
@@ -239,6 +245,21 @@ The PostgreSQL software includes:
       src/backend/libpq/sha2.h
       see licenses/LICENSE-bsd3-sha2.txt
 
+      gpMgmt/bin/pythonSrc/ext/psutil-5.7.0.tar.gz
+      see licenses/LICENSE-bsd3-psutil.txt
+
+----------------------------
+    MIT LICENSE
+
+      gpMgmt/bin/pythonSrc/ext/PyYAML-5.3.1.tar.gz
+      see licenses/LICENSE-pyyaml.txt
+
+      gpMgmt/bin/pythonSrc/ext/setuptools-36.6.0.tar.gz
+      see licenses/LICENSE-setuptools.txt
+
+      gpMgmt/bin/pythonSrc/ext/parse-1.8.2.tar.gz
+      see licenses/LICENSE-parse.txt
+
 ----------------------------
    Internet Systems Consortium/Internet Software Consortium (ISC) LICENSE
 
@@ -335,4 +356,4 @@ This product includes code from pg_cron under PostgreSQL license:
    /src/backend/task/job_metadata.c
    /src/backend/task/pg_cron.c
 
-   see licenses/LICENSE-citusdata.txt
+   see licenses/LICENSE-citusdata.txt

README.md

@@ -23,7 +23,9 @@
   <img alt="Coverity Scan Build Status"
        src="https://scan.coverity.com/projects/31473/badge.svg"/>
 </a>
-
+<a href="https://sonarcloud.io/summary/new_code?id=apache_cloudberry">
+  <img alt="SonarQube Cloud" src="https://sonarcloud.io/images/project_badges/sonarcloud-highlight.svg" width="100px">
+</a>
 ---------
 
 ## Introduction