Skip to content

Commit 2a808bf

Browse files
SkyFan2002SkyFan2002
authored
Merge branch 'main' into shuffle_rf
2 parents 95f767b + 18ce7dd commit 2a808bf

File tree

364 files changed

+8551
-4659
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

364 files changed

+8551
-4659
lines changed

‎.github/actions/build_linux/action.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ runs:
9090
- name: Check Binary Info
9191
shell: bash
9292
run: |
93-
grep ' Compiled in ' target/sccache.log
93+
grep ' Compiled in ' target/sccache.log || true
9494
readelf -p .comment ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
9595
ldd ./target/${{ inputs.target }}/${{ env.BUILD_PROFILE }}/databend-query || true
9696

‎.github/actions/pack_binaries/action.yml

+6-6
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,9 @@ runs:
7272
name: sha256sums-${{ inputs.category }}-${{ inputs.target }}
7373
path: sha256-${{ steps.pack_binaries.outputs.pkg_name }}.txt
7474
retention-days: 1
75-
- name: post binaries
76-
uses: actions/upload-artifact@v4
77-
with:
78-
name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
79-
path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
80-
retention-days: 1
75+
# - name: post binaries
76+
# uses: actions/upload-artifact@v4
77+
# with:
78+
# name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
79+
# path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
80+
# retention-days: 1

‎.github/actions/pack_deb/action.yml

+18-33
Original file line numberDiff line numberDiff line change
@@ -42,49 +42,34 @@ runs:
4242
export version="${{ inputs.version }}"
4343
case "${{ inputs.arch }}" in
4444
x86_64)
45-
echo "deb_arch=amd64" >> $GITHUB_OUTPUT
45+
echo "arch=amd64" >> $GITHUB_OUTPUT
4646
;;
4747
aarch64)
48-
echo "deb_arch=arm64" >> $GITHUB_OUTPUT
48+
echo "arch=arm64" >> $GITHUB_OUTPUT
49+
;;
50+
*)
51+
echo "Unsupported arch: ${{ inputs.arch }}"
52+
exit 1
4953
;;
5054
esac
51-
deb_version=${version/-/.}
52-
echo "deb_version=${deb_version/v/}" >> $GITHUB_OUTPUT
53-
mkdir -p distro/{bin,configs,systemd,scripts}
55+
deb_version=${version/-/'~'}
56+
echo "version=${deb_version/v/}" >> $GITHUB_OUTPUT
57+
mkdir -p distro/{bin,configs,systemd,scripts,pkg}
5458
cp ./scripts/distribution/systemd/databend-* distro/systemd/
5559
cp ./scripts/distribution/configs/databend-* distro/configs/
5660
cp ./scripts/distribution/release-readme.txt distro/readme.txt
5761
cp -r ./scripts/distribution/package-scripts/* distro/scripts/
5862
59-
- name: Build Databend Query Package
60-
shell: bash
61-
run: |
62-
export path="distro"
63-
export version="${{ inputs.version }}"
64-
export deb_version="${{ steps.info.outputs.deb_version }}"
65-
export deb_arch="${{ steps.info.outputs.deb_arch }}"
66-
pkg_name="databend-query_${deb_version}_${deb_arch}.deb"
67-
nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query.yaml)
68-
69-
- name: Build Databend Meta Package
63+
- name: Build Databend Packages
7064
shell: bash
7165
run: |
72-
export path="distro"
73-
export version="${{ inputs.version }}"
74-
export deb_version="${{ steps.info.outputs.deb_version }}"
75-
export deb_arch="${{ steps.info.outputs.deb_arch }}"
76-
pkg_name="databend-meta_${deb_version}_${deb_arch}.deb"
77-
nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-meta.yaml)
78-
79-
- name: Build Databend Debug Package
80-
shell: bash
81-
run: |
82-
export path="distro"
83-
export version="${{ inputs.version }}"
84-
export deb_version="${{ steps.info.outputs.deb_version }}"
85-
export deb_arch="${{ steps.info.outputs.deb_arch }}"
86-
pkg_name="databend-query-dbg_${deb_version}_${deb_arch}.deb"
87-
nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query-dbg.yaml)
66+
pkgs="query meta query-dbg"
67+
for pkg in $pkgs; do
68+
yq -i '.arch = "${{ steps.info.outputs.arch }}"' scripts/distribution/nfpm-${pkg}.yaml
69+
yq -i '.version = "${{ steps.info.outputs.version }}"' scripts/distribution/nfpm-${pkg}.yaml
70+
nfpm pkg --packager deb --target distro/pkg -f scripts/distribution/nfpm-${pkg}.yaml
71+
done
72+
ls -lh distro/pkg
8873
8974
- name: Update release to github
9075
shell: bash
@@ -94,6 +79,6 @@ runs:
9479
run: |
9580
version="${{ inputs.version }}"
9681
# name looks like: `databend_0.8.144~nightly_amd64.deb`
97-
for pkg in $(ls distro/*.deb); do
82+
for pkg in $(ls distro/pkg/*.deb); do
9883
gh release upload ${version} $pkg --clobber
9984
done

‎.github/actions/publish_deb/action.yml

+9-4
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,8 @@ runs:
3131
# FIXME: tmp disable dbg package
3232
assets=$(gh release view --repo databendlabs/databend --json assets | jq -r '.assets[] | select(.contentType == "application/x-debian-package") | .url' | grep -v 'dbg')
3333
for asset in $assets; do
34+
echo "--> downloading $asset"
3435
wget -q $asset
35-
asset_name=$(basename $asset)
36-
reprepro includedeb stable ${asset_name}
3736
done
3837
3938
- name: Add BendSQL Packages
@@ -44,11 +43,17 @@ runs:
4443
echo "version: $version"
4544
assets=$(gh release view --repo databendlabs/bendsql --json assets | jq -r '.assets[] | select(.contentType == "application/x-debian-package") | .url')
4645
for asset in $assets; do
46+
echo "--> downloading $asset"
4747
wget -q $asset
48-
asset_name=$(basename $asset)
49-
reprepro includedeb stable ${asset_name}
5048
done
5149
50+
- name: Include DEB Packages
51+
shell: bash
52+
working-directory: scripts/distribution/deb
53+
run: |
54+
ls -lh
55+
reprepro includedeb stable *.deb
56+
5257
- name: Publish to S3
5358
shell: bash
5459
working-directory: scripts/distribution/deb

‎.github/actions/setup_build_tool/action.yml

+9-2
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ runs:
4949
export SCCACHE_S3_KEY_PREFIX="cache/sccache/"
5050
export SCCACHE_S3_USE_SSL="true"
5151
export SCCACHE_ERROR_LOG=/workspace/target/sccache.log
52-
export SCCACHE_LOG=debug
52+
export SCCACHE_LOG=info
5353
EOF
5454
;;
5555
@@ -62,7 +62,7 @@ runs:
6262
export SCCACHE_GCS_BUCKET=databend-ci
6363
export SCCACHE_GCS_KEY_PREFIX="cache/sccache/"
6464
export SCCACHE_ERROR_LOG=/workspace/target/sccache.log
65-
export SCCACHE_LOG=debug
65+
export SCCACHE_LOG=info
6666
EOF
6767
;;
6868
@@ -101,6 +101,13 @@ runs:
101101
shell: bash
102102
run: df -h
103103

104+
- name: check system info
105+
shell: bash
106+
run: |
107+
uname -a
108+
cat /proc/cpuinfo
109+
free -h
110+
104111
# - uses: everpcpc/actions-cache@v1
105112
# if: env.RUNNER_PROVIDER == 'aws'
106113
# env:

‎.github/actions/setup_test/action.yml

+13
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,16 @@ runs:
4949
shell: bash
5050
run: |
5151
echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.LD_LIBRARY_PATH }}" >> $GITHUB_ENV
52+
53+
- name: check docker info
54+
shell: bash
55+
run: |
56+
docker info
57+
docker version
58+
docker ps -a
59+
docker images
60+
61+
- name: setup env
62+
shell: bash
63+
run: |
64+
echo "RUST_LOG=info" >> $GITHUB_ENV

‎.github/actions/test_compat_fuse/action.yml

+11-14
Original file line numberDiff line numberDiff line change
@@ -5,25 +5,22 @@ runs:
55
steps:
66
- uses: ./.github/actions/setup_test
77
with:
8-
path: ./bins/current
8+
path: ./bins/current/bin
99
artifacts: sqllogictests,meta,query
1010

1111
- name: Test compatibility
1212
shell: bash
13-
# test-*.sh <old-query-ver> <meta-ver> <test-suite>
14-
# The `meta-ver` must be compatible with `<old-query-ver>` and the latest version.
15-
#
16-
# `meta-ver` is not the target to test, just choose one that is compatible with `<old-query-ver>` and the latest query.
1713
run: |
18-
bash ./tests/compat_fuse/test_compat_fuse.sh 1.2.46 1.2.527 base
19-
bash ./tests/compat_fuse/test_compat_fuse.sh 1.2.241 1.2.527 revoke
20-
bash ./tests/compat_fuse/test_compat_fuse.sh 1.2.306 1.2.527 rbac
21-
bash ./tests/compat_fuse/test_compat_fuse.sh 1.2.307 1.2.527 rbac
22-
bash ./tests/compat_fuse/test_compat_fuse.sh 1.2.318 1.2.527 rbac
23-
bash ./tests/compat_fuse/test_compat_fuse_forward.sh 1.2.307 1.2.527 rbac
24-
bash ./tests/compat_fuse/test_compat_fuse_forward.sh 1.2.318 1.2.527 rbac
25-
bash ./tests/compat_fuse/test_compat_fuse.sh 1.2.680 1.2.680 udf
26-
bash ./tests/compat_fuse/test_compat_fuse_forward.sh 1.2.680 1.2.680 udf
14+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version 1.2.46 --reader-version current --meta-versions 1.2.527 1.2.677 --logictest-path base
15+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version 1.2.241 --reader-version current --meta-versions 1.2.527 1.2.677 --logictest-path revoke
16+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version 1.2.306 --reader-version current --meta-versions 1.2.527 1.2.677 --logictest-path rbac
17+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version 1.2.307 --reader-version current --meta-versions 1.2.527 1.2.677 --logictest-path rbac
18+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version 1.2.318 --reader-version current --meta-versions 1.2.527 1.2.677 --logictest-path rbac
19+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version 1.2.680 --reader-version current --meta-versions 1.2.527 1.2.680 --logictest-path udf
20+
21+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version current --reader-version 1.2.307 --meta-versions 1.2.677 --logictest-path rbac
22+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version current --reader-version 1.2.318 --meta-versions 1.2.677 --logictest-path rbac
23+
bash ./tests/compat_fuse/test_compat_fuse.sh --writer-version current --reader-version 1.2.680 --meta-versions 1.2.680 --logictest-path udf
2724
- name: Upload failure
2825
if: failure()
2926
uses: ./.github/actions/artifact_failure

‎.github/actions/test_unit/action.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ runs:
1212
- shell: bash
1313
run: |
1414
cargo -Zgitoxide=fetch -Zgit=shallow-index,shallow-deps nextest run --no-fail-fast --hide-progress-bar
15-
grep ' Compiled in ' target/sccache.log
15+
grep ' Compiled in ' target/sccache.log || true
1616
env:
1717
RUST_TEST_THREADS: "8"
1818
RUST_LOG: ERROR

‎.github/scripts/generate_release_note.js

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
const fs = require("fs");
2+
3+
function escapeHtml(text) {
4+
const map = {
5+
"&": "&amp;",
6+
"<": "&lt;",
7+
">": "&gt;",
8+
'"': "&quot;",
9+
"'": "&#039;",
10+
};
11+
return text.replace(/[&<>"']/g, (m) => map[m]);
12+
}
13+
14+
module.exports = async ({ github, context, core }) => {
15+
const { VERSION, DATE } = process.env;
16+
fs.mkdirSync("docs/release-stable", { recursive: true });
17+
const df = `docs/release-stable/${DATE}_${VERSION}.md`;
18+
19+
const releases = await github.rest.repos.listReleases({
20+
owner: "databendlabs",
21+
repo: "databend",
22+
});
23+
const release = releases.data.find((r) => r.name === VERSION);
24+
if (!release) {
25+
core.setFailed(`Release ${VERSION} not found`);
26+
return;
27+
}
28+
29+
let body = release.body;
30+
body = body.split("\n").slice(1).join("\n");
31+
body = "---\n" + body;
32+
body = body.replace(/^--$/gm, "---");
33+
body = body.replace(/^asset:.*$/gm, "");
34+
body = body.replace(
35+
/https:\/\/github\.com\/databendlabs\/databend\/pull\/([0-9]+)/g,
36+
"[#$1](https://github.com/databendlabs/databend/pull/$1)"
37+
);
38+
body = escapeHtml(body);
39+
40+
fs.writeFileSync(df, body);
41+
};

‎.github/workflows/release.yml

+8-8
Original file line numberDiff line numberDiff line change
@@ -92,16 +92,16 @@ jobs:
9292
shell: bash
9393
run: echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
9494
- name: Generate Release Note
95+
uses: actions/github-script@v7
9596
env:
96-
GH_TOKEN: ${{ github.token }}
97+
VERSION: ${{ needs.create_release.outputs.version }}
98+
DATE: ${{ steps.date.outputs.DATE }}
99+
with:
100+
script: |
101+
const script = require('./.github/scripts/generate_release_note.js')
102+
await script({ github, context, core })
103+
- name: Add Release Note
97104
run: |
98-
mkdir -p docs/release-stable
99-
df="docs/release-stable/${{ steps.date.outputs.DATE }}_${{ needs.create_release.outputs.version }}.md"
100-
echo "---" > $df
101-
gh release view --repo databendlabs/databend ${{ needs.create_release.outputs.version }} >> $df
102-
sed -i -E 's/^--$/---/g' $df
103-
sed -i -E '/^asset:/d' $df
104-
sed -i -E 's_https://github.com/databendlabs/databend/pull/([0-9]+)_[#\1](https://github.com/databendlabs/databend/pull/\1)_g' $df
105105
git add docs/release-stable
106106
git status
107107
- uses: peter-evans/create-pull-request@v4

‎.github/workflows/reuse.linux.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,7 @@ jobs:
158158
steps:
159159
- uses: actions/checkout@v4
160160
- uses: ./.github/actions/test_stateless_standalone_linux
161-
timeout-minutes: 15
161+
timeout-minutes: 18
162162

163163
test_stateless_cluster:
164164
runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]
@@ -170,7 +170,7 @@ jobs:
170170
runner_provider: ${{ inputs.runner_provider }}
171171
type: ${{ inputs.license_type }}
172172
- uses: ./.github/actions/test_stateless_cluster_linux
173-
timeout-minutes: 15
173+
timeout-minutes: 18
174174

175175
test_stateful_standalone:
176176
runs-on: [self-hosted, X64, Linux, 2c8g, "${{ inputs.runner_provider }}"]

0 commit comments

Comments
 (0)