Skip to content
Test

Test #3581

Sign in to view logs

Test

Test #3581

Workflow file for this run

.github/workflows/main.yaml at 6a7b0ca
name: Test
on:
push:
branches:
- main
- v0.5
paths-ignore:
- '**/*.md'
- 'docs/**'
- 'scripts/**'
- '.github/mergify.yml'
pull_request:
branches:
- main
- v0.5
paths-ignore:
- '**/*.md'
- 'docs/**'
- 'scripts/**'
- '.github/mergify.yml'
workflow_dispatch:
schedule:
- cron: '0 0 * * *'
env:
go-version: 1.26
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
lint:
name: Lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Go ${{ env.go-version }}
uses: actions/setup-go@v7
with:
go-version: ${{ env.go-version }}
cache: true
- name: golangci-lint
uses: golangci/golangci-lint-action@v9
with:
version: latest
unit-test-go:
name: Unit test go
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Go ${{ env.go-version }}
uses: actions/setup-go@v7
with:
go-version: ${{ env.go-version }}
cache: true
- name: Unit test with coverage
run: go test --race -coverprofile=coverage.out -covermode=atomic ./...
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v7
with:
files: coverage.out
fail_ci_if_error: true
token: ${{ secrets.CODECOV_TOKEN }}
check-sample-configs:
name: Check sample configs
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- name: Set up Go ${{ env.go-version }}
uses: actions/setup-go@v7
with:
go-version: ${{ env.go-version }}
cache: true
# The samples in configs/ are what people copy, so a broken one is
# otherwise only found by the person copying it.
- name: Load every sample config
run: make check-configs
test-backup-restore-cross-version:
needs: unit-test-go
name: Backup and restore cross version
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
another_milvus_mode: [standalone]
source_image_tag: [v2.2.16, v2.3.22, v2.4.23, v2.5.20, 2.6-latest]
target_image_tag: [2.6-latest]
exclude:
- source_image_tag: 2.6-latest
target_image_tag: 2.6-latest
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.10
uses: actions/setup-python@v7
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v7
name: Set up Go ${{ env.go-version }}
with:
go-version: ${{ env.go-version }}
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.source_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Export docker-compose status after deploy
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml ps -a || true
echo "=== Standalone Container Full Logs ==="
sudo docker compose -f docker-compose.yml logs standalone || true
echo "=== All Containers Status ==="
sudo docker ps -a || true
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: List MinIO files before backup
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
sudo apt-get install -y tree > /dev/null 2>&1 || true
echo "=== MinIO volume tree (before backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup
./milvus-backup list
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
- name: List MinIO files after backup
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
echo "=== MinIO volume tree (after backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup delete -n my_backup
./milvus-backup list
- name: Uninstall Milvus
timeout-minutes: 5
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
sudo docker compose -f docker-compose.yml down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.target_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
cat custom_config.yaml || true
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Export docker-compose status after another deploy
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml ps -a || true
echo "=== Standalone Container Full Logs ==="
sudo docker compose -f docker-compose.yml logs standalone || true
echo "=== All Containers Status ==="
sudo docker ps -a || true
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
shell: bash
run: |
pwd
ls -l
tree
- name: Restore backup from another Milvus
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data from another Milvus
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
test-backup-restore-local-storage:
needs: unit-test-go
# Milvus standalone with COMMON_STORAGETYPE=local keeps its data on the
# local filesystem (localStorage.path, default /var/lib/milvus/data). This
# job proves milvus-backup can read that directory directly and back it up
# to MinIO, then restores it into a fresh Milvus backed by MinIO or by local
# storage, resolving the restore temp files at the directory the process
# sees in the local case (issue #964).
name: Backup and restore local storage Milvus
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image_tag: [v2.5.20, 2.6-latest]
restore_target: [minio, local]
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.10
uses: actions/setup-python@v7
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v7
name: Set up Go ${{ env.go-version }}
with:
go-version: ${{ env.go-version }}
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy with local storage
timeout-minutes: 15
shell: bash
working-directory: deployment/standalone
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml -f docker-compose.local.yaml up -d --wait
sudo docker compose -f docker-compose.yml -f docker-compose.local.yaml ps -a
- name: Export docker-compose status after deploy
if: ${{ always() }}
shell: bash
working-directory: deployment/standalone
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml -f docker-compose.local.yaml ps -a || true
echo "=== Standalone Container Full Logs ==="
sudo docker compose -f docker-compose.yml -f docker-compose.local.yaml logs standalone || true
echo "=== All Containers Status ==="
sudo docker ps -a || true
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: Fix permissions after data preparation
shell: bash
working-directory: deployment/standalone
run: |
# The container writes binlogs as root; the backup binary runs as the
# runner user and reads the local directory directly.
sudo chmod -R 777 volumes
- name: List milvus local data before backup
shell: bash
working-directory: deployment/standalone
run: |
sudo apt-get install -y tree > /dev/null 2>&1 || true
echo "=== milvus local data tree (before backup) ==="
sudo tree volumes/milvus/data | head -500
- name: Configure backup.yaml for local storage
timeout-minutes: 1
shell: bash
run: |
yq -i '.log.level = "debug"' configs/backup.yaml
yq -i '.milvus.storage.provider = "local"' configs/backup.yaml
# Host directory backing Milvus localStorage.path (mounted at
# /var/lib/milvus in the container).
yq -i '.milvus.storage.rootPath = "deployment/standalone/volumes/milvus/data"' configs/backup.yaml
yq -i '.milvus.storage.bucketName = ""' configs/backup.yaml
cat configs/backup.yaml
- name: Backup
timeout-minutes: 10
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup
./milvus-backup list
- name: List MinIO files after backup
if: ${{ always() }}
shell: bash
working-directory: deployment/standalone
run: |
echo "=== MinIO volume tree (after backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/standalone/volumes/minio/a-bucket/backup local-backup
- name: Uninstall Milvus
timeout-minutes: 5
shell: bash
working-directory: deployment/standalone
run: |
sudo docker compose -f docker-compose.yml -f docker-compose.local.yaml down
sudo rm -rf volumes
- name: Configure backup.yaml for ${{ matrix.restore_target }} storage target
timeout-minutes: 1
shell: bash
run: |
if [ "${{ matrix.restore_target }}" = "local" ]; then
yq -i '.milvus.storage.provider = "local"' configs/backup.yaml
# Host directory backing the target Milvus localStorage.path, and the
# path the container process resolves it at.
yq -i '.milvus.storage.rootPath = "deployment/standalone/volumes/milvus/data"' configs/backup.yaml
yq -i '.milvus.storage.localPath = "/var/lib/milvus/data"' configs/backup.yaml
yq -i '.milvus.storage.bucketName = ""' configs/backup.yaml
else
yq -i '.milvus.storage.provider = "minio"' configs/backup.yaml
yq -i '.milvus.storage.rootPath = "files"' configs/backup.yaml
yq -i '.milvus.storage.bucketName = "a-bucket"' configs/backup.yaml
fi
cat configs/backup.yaml
- name: Deploy target Milvus with ${{ matrix.restore_target }} storage
timeout-minutes: 15
shell: bash
working-directory: deployment/standalone
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag 2.6-latest) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
if [ "${{ matrix.restore_target }}" = "local" ]; then
sudo docker compose -f docker-compose.yml -f docker-compose.local.yaml up -d --wait
sudo docker compose -f docker-compose.yml -f docker-compose.local.yaml ps -a
else
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
fi
- name: Fix permissions for restore
if: matrix.restore_target == 'local'
shell: bash
working-directory: deployment/standalone
run: |
# The container initializes its storage directory as root; the restore
# binary runs as the runner user and must write restore temp files
# into the same directory for the container to import.
sudo chmod -R 777 volumes
- name: Copy Backup to target MinIO
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/standalone/volumes/minio/a-bucket/backup
sudo cp -r local-backup/my_backup deployment/standalone/volumes/minio/a-bucket/backup
- name: Restore backup
timeout-minutes: 10
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
test-backup-restore-after-upgrade:
needs: unit-test-go
name: Backup and restore after upgrade
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
another_milvus_mode: [standalone]
source_image_tag: [2.5-latest]
target_image_tag: [2.6-latest]
exclude:
- source_image_tag: 2.6-latest
target_image_tag: 2.6-latest
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.10
uses: actions/setup-python@v7
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v7
name: Set up Go ${{ env.go-version }}
with:
go-version: ${{ env.go-version }}
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.source_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Export docker-compose status after initial deploy
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml ps -a || true
echo "=== Standalone Container Full Logs ==="
sudo docker compose -f docker-compose.yml logs standalone || true
echo "=== All Containers Status ==="
sudo docker ps -a || true
- name: Prepare data stage 1
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py --stage 1
- name: Upgrade Milvus
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
sudo docker compose -f docker-compose.yml down
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.target_image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
cat custom_config.yaml || true
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Export docker-compose status after upgrade
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml ps -a || true
echo "=== Standalone Container Full Logs ==="
sudo docker compose -f docker-compose.yml logs standalone || true
echo "=== All Containers Status ==="
sudo docker ps -a || true
- name: Prepare data stage 2
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py --stage 2
- name: List MinIO files before backup
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
sudo apt-get install -y tree > /dev/null 2>&1 || true
echo "=== MinIO volume tree (before backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup
./milvus-backup list
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
# --filter matches the collection name after -s is applied, so it
# names hello_milvus_recover rather than hello_milvus.
./milvus-backup restore -n my_backup --filter hello_milvus_recover -s _recover
./milvus-backup restore -n my_backup --filter hello_milvus2_recover -s _recover
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
- name: List MinIO files after backup
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
echo "=== MinIO volume tree (after backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
test-backup-restore-cli:
needs: unit-test-go
name: Backup and restore cli
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
another_milvus_mode: [standalone]
image_tag: [2.6-latest]
# mq_type: [pulsar, kafka] # TODO: add pulsar and kafka
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.10
uses: actions/setup-python@v7
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v7
name: Set up Go ${{ env.go-version }}
with:
go-version: ${{ env.go-version }}
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
cat custom_config.yaml || true
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Export docker-compose status after deploy
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml ps -a || true
echo "=== Standalone Container Full Logs ==="
sudo docker compose -f docker-compose.yml logs standalone || true
echo "=== All Containers Status ==="
sudo docker ps -a || true
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py
- name: List MinIO files before backup
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
sudo apt-get install -y tree > /dev/null 2>&1 || true
echo "=== MinIO volume tree (before backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup
./milvus-backup list
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
- name: List MinIO files after backup
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
echo "=== MinIO volume tree (after backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Save Backup
timeout-minutes: 5
shell: bash
run: |
sudo cp -r deployment/${{ matrix.milvus_mode }}/volumes/minio/a-bucket/backup ${{ matrix.milvus_mode }}-backup
- name: delete backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup delete -n my_backup
./milvus-backup list
- name: Uninstall Milvus
timeout-minutes: 5
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
sudo docker compose -f docker-compose.yml down
sudo rm -rf volumes
- name: Deploy Another Milvus
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Export docker-compose status after another deploy
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.another_milvus_mode }}
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml ps -a || true
echo "=== Standalone Container Full Logs ==="
sudo docker compose -f docker-compose.yml logs standalone || true
echo "=== All Containers Status ==="
sudo docker ps -a || true
- name: Copy Backup to Another Milvus
timeout-minutes: 5
shell: bash
run: |
sudo mkdir -p deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
sudo cp -r ${{ matrix.milvus_mode }}-backup/my_backup deployment/${{ matrix.another_milvus_mode }}/volumes/minio/a-bucket/backup
- name: List backup from another Milvus
timeout-minutes: 5
working-directory: deployment/${{ matrix.another_milvus_mode }}/volumes/minio
shell: bash
run: |
pwd
ls -l
tree
- name: Restore backup from another Milvus
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover
- name: Verify data from another Milvus
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py
test-backup-restore-secondary:
needs: unit-test-go
name: Backup and restore secondary (${{ matrix.scenario }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
# Three scenarios:
# - same-version: source and target are both the latest 2.6.
# - same-version-master: source and target are both the latest master,
# the only line that has import two-phase commit. #1080 dropped master
# from CI because red master jobs hid regressions on the released
# line; this brings it back for secondary restore alone, next to the
# 2.6 scenario rather than in place of it, so the 2PC path is covered
# without master failures masking a 2.6 one.
# - cross-version-upgrade: source data is created on a pre-#46419 Milvus
# (v2.6.7, where the auto-appended $meta field has no Nullable nor
# DefaultValue), then the cluster is upgraded to a CDC-capable image
# (2.6-latest, since CDC support landed in v2.6.11+) before backup
# and secondary restore. This reproduces the schema misalignment from
# zilliztech/milvus-backup#1013 — etcd-schema-verify is expected to
# fail on the $meta field's nullable/default_value until the bug is
# fixed.
include:
- scenario: same-version
prepare_image_tag: 2.6-latest
run_image_tag: 2.6-latest
- scenario: same-version-master
prepare_image_tag: master-latest
run_image_tag: master-latest
- scenario: cross-version-upgrade
prepare_image_tag: v2.6.7
run_image_tag: 2.6-latest
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.10
uses: actions/setup-python@v7
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v7
name: Set up Go ${{ env.go-version }}
with:
go-version: ${{ env.go-version }}
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
go get
go build
go build -o inittarget ./cmd/inittarget
go build -o etcd-schema-verify ./cmd/etcd-schema-verify
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Deploy dual Milvus (prepare image)
timeout-minutes: 15
shell: bash
working-directory: deployment/secondary
run: |
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.prepare_image_tag }}) && echo $tag
yq -i ".services.upstream-standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
yq -i ".services.downstream-standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
mkdir -p volumes/etcd volumes/minio volumes/upstream volumes/downstream volumes/upstream-runtime volumes/downstream-runtime
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Export docker-compose status after deploy
if: ${{ always() }}
shell: bash
working-directory: deployment/secondary
run: |
echo "=== Docker Compose Status ==="
sudo docker compose -f docker-compose.yml ps -a || true
echo "=== Upstream Container Logs ==="
sudo docker compose -f docker-compose.yml logs upstream-standalone || true
echo "=== Downstream Container Logs ==="
sudo docker compose -f docker-compose.yml logs downstream-standalone || true
- name: Configure backup.yaml for upstream
timeout-minutes: 1
shell: bash
run: |
yq -i '.log.level = "debug"' configs/backup.yaml
yq -i '.milvus.grpc.port = 19530' configs/backup.yaml
yq -i '.milvus.storage.bucketName = "milvus-backup-test"' configs/backup.yaml
yq -i '.milvus.storage.rootPath = "backup-test-upstream"' configs/backup.yaml
yq -i '.backup.storage.bucketName = "milvus-backup-test"' configs/backup.yaml
yq -i '.milvus.replicate.rpcChannelName = "backup-test-upstream-replicate-msg"' configs/backup.yaml
yq -i '.milvus.etcd.rootPath = "backup-test-upstream"' configs/backup.yaml
cat configs/backup.yaml
- name: Prepare data on upstream
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py --uri http://127.0.0.1:19530
python example/prepare_data.py --uri http://127.0.0.1:19530 --scenario added
- name: Upgrade dual Milvus to run image
if: ${{ matrix.prepare_image_tag != matrix.run_image_tag }}
timeout-minutes: 15
shell: bash
working-directory: deployment/secondary
run: |
# Stop containers but keep bind-mounted volumes (./volumes/*) so the
# collection schema written by the prepare image — including its
# version-specific $meta attributes — survives the upgrade.
sudo docker compose -f docker-compose.yml down
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.run_image_tag }}) && echo $tag
yq -i ".services.upstream-standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
yq -i ".services.downstream-standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Create index and load on upstream
timeout-minutes: 3
shell: bash
run: |
python -c "
from pymilvus import connections, Collection
connections.connect('default', uri='http://127.0.0.1:19530')
index = {'index_type': 'IVF_FLAT', 'metric_type': 'L2', 'params': {'nlist': 128}}
for name in ['hello_milvus', 'hello_milvus2', 'hello_milvus_dynamic', 'hello_milvus_added']:
c = Collection(name)
c.create_index('embeddings', index)
c.load()
print(f'{name}: loaded, entities={c.num_entities}')
"
- name: List MinIO files before backup
shell: bash
working-directory: deployment/secondary
run: |
sudo apt-get install -y tree > /dev/null 2>&1 || true
echo "=== MinIO volume tree (before backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Create backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
# Pin binlog until ExportSnapshot lands on the milvus master line: master
# reports the snapshot feature, so auto resolves to the snapshot format
# there, which its export RPC does not implement yet.
./milvus-backup create -n my_backup --backup_index_extra --format=binlog
./milvus-backup list
- name: Init downstream replicate configuration
timeout-minutes: 1
shell: bash
run: |
./inittarget --target downstream \
--upstream-uri upstream-standalone:19530 \
--downstream-uri downstream-standalone:19530
- name: Configure backup.yaml for downstream
timeout-minutes: 1
shell: bash
run: |
yq -i '.milvus.grpc.port = 19500' configs/backup.yaml
cat configs/backup.yaml
- name: Restore secondary
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore secondary -n my_backup \
--source_cluster_id backup-test-upstream \
--target_cluster_id backup-test-downstream
- name: Verify etcd schema alignment
timeout-minutes: 1
shell: bash
run: |
set -o pipefail
for coll in hello_milvus hello_milvus2 hello_milvus_dynamic hello_milvus_added; do
echo "=== Verifying schema: ${coll} ==="
./etcd-schema-verify \
-src-endpoints localhost:2379 \
-dst-endpoints localhost:2379 \
-src-root-path backup-test-upstream \
-dst-root-path backup-test-downstream \
-collection "${coll}" | tee /tmp/schema_verify_${coll}.json
echo "Schema aligned for ${coll}"
done
- name: Init upstream replicate configuration
timeout-minutes: 1
shell: bash
run: |
./inittarget --target upstream \
--upstream-uri upstream-standalone:19530 \
--downstream-uri downstream-standalone:19530
- name: Verify data on downstream
timeout-minutes: 5
shell: bash
run: |
python example/secondary/verify_data.py --uri http://127.0.0.1:19500
- name: List MinIO files after backup
if: ${{ always() }}
shell: bash
working-directory: deployment/secondary
run: |
echo "=== MinIO volume tree (after backup) ==="
sudo tree volumes/minio/ -I '.minio.sys' | head -500
- name: Export logs
if: ${{ always() }}
shell: bash
working-directory: deployment/secondary
run: |
mkdir -p /tmp/ci_logs
sudo docker compose -f docker-compose.yml logs upstream-standalone > /tmp/ci_logs/upstream.log 2>&1 || true
sudo docker compose -f docker-compose.yml logs downstream-standalone > /tmp/ci_logs/downstream.log 2>&1 || true
sudo docker compose -f docker-compose.yml logs etcd > /tmp/ci_logs/etcd.log 2>&1 || true
sudo docker compose -f docker-compose.yml logs minio > /tmp/ci_logs/minio.log 2>&1 || true
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v7
with:
name: secondary-restore-logs-${{ matrix.scenario }}
path: |
/tmp/ci_logs
/tmp/schema_verify_*.json
./server.log
test-backup-restore-with-rbac-config:
needs: unit-test-go
name: Backup and restore with rbac config
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
deploy_tools: [helm]
milvus_mode: [standalone]
image_tag: [2.6-latest]
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.10
uses: actions/setup-python@v7
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v7
with:
go-version: ${{ env.go-version }}
cache: "true"
- name: Creating kind cluster
if: ${{ matrix.deploy_tools == 'helm' }}
uses: helm/kind-action@v1.14.0
- name: Build
timeout-minutes: 5
shell: bash
run: |
yq -i '.log.level = "debug"' configs/backup.yaml
yq -i '.milvus.storage.bucketName = "milvus-bucket"' configs/backup.yaml
yq -i '.milvus.storage.rootPath = "file"' configs/backup.yaml
yq -i '.backup.storage.port = 9010' configs/backup.yaml
yq -i '.backup.storage.bucketName = "milvus-bucket"' configs/backup.yaml
yq -i '.backup.storage.rootPath = "backup"' configs/backup.yaml
yq -i '.transfer.mode = "streaming"' configs/backup.yaml
cat configs/backup.yaml || true
go get
go build
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
helm repo add milvus https://zilliztech.github.io/milvus-helm
helm repo update
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".image.all.tag=\"${tag}\"" rbac-values.yaml
helm install --wait --debug --timeout 600s milvus-backup milvus/milvus -f rbac-values.yaml
helm install --wait --debug --timeout 600s milvus-restore milvus/milvus -f rbac-values.yaml
kubectl get pods
kubectl port-forward service/milvus-backup 19530 >/dev/null 2>&1 &
kubectl port-forward service/milvus-backup-minio 9000 >/dev/null 2>&1 &
kubectl port-forward service/milvus-backup 9091 >/dev/null 2>&1 &
kubectl port-forward service/milvus-restore 19531:19530 >/dev/null 2>&1 &
kubectl port-forward service/milvus-restore-minio 9010:9000 >/dev/null 2>&1 &
sleep 10
nc -vz 127.0.0.1 19530
nc -vz 127.0.0.1 9000
nc -vz 127.0.0.1 9091
nc -vz 127.0.0.1 19531
nc -vz 127.0.0.1 9010
sleep 10
kubectl get pods -n default | grep milvus-backup
kubectl get pods -n default | grep milvus-restore
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
fi
- name: Prepare data
timeout-minutes: 5
shell: bash
run: |
python example/prepare_data.py --uri http://127.0.0.1:19530
python example/prepare_rbac.py --uri http://127.0.0.1:19530
- name: List MinIO files before backup
shell: bash
run: |
MINIO_POD=$(kubectl get pods -o name | grep milvus-backup.*minio | head -1)
echo "=== MinIO file tree (before backup) ==="
kubectl exec $MINIO_POD -- sh -c "apt-get install -y tree > /dev/null 2>&1 || true; tree /export/ -I '.minio.sys' | head -500"
- name: Backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup check
./milvus-backup list
./milvus-backup create -n my_backup --rbac
./milvus-backup list
- name: Modify Restore Config
timeout-minutes: 5
shell: bash
run: |
yq -i '.milvus.grpc.port = 19531' configs/backup.yaml
yq -i '.milvus.storage.port = 9010' configs/backup.yaml
- name: Restore backup
timeout-minutes: 5
shell: bash
run: |
./milvus-backup restore -n my_backup -s _recover --rbac
- name: Verify data
timeout-minutes: 5
shell: bash
run: |
python example/verify_data.py --uri http://127.0.0.1:19531
python example/verify_rbac.py --uri http://127.0.0.1:19531
- name: List MinIO files after backup
if: ${{ always() }}
shell: bash
run: |
MINIO_POD=$(kubectl get pods -o name | grep milvus-backup.*minio | head -1)
echo "=== MinIO file tree (after backup) ==="
kubectl exec $MINIO_POD -- sh -c "apt-get install -y tree > /dev/null 2>&1 || true; tree /export/ -I '.minio.sys' | head -500"
- name: Export logs
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "helm" ]; then
kubectl get pods
bash ../../scripts/export_log_k8s.sh default milvus-backup logs
bash ../../scripts/export_log_k8s.sh default milvus-restore logs
fi
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v7
with:
name: custom-config-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}
path: |
./logs
./server.log
/tmp/ci_logs
deployment/${{ matrix.milvus_mode }}/logs
test-backup-restore-api:
name: Backup and restore api
runs-on: ubuntu-latest
needs: test-backup-restore-cli
strategy:
fail-fast: false
matrix:
deploy_tools: [docker-compose]
milvus_mode: [standalone]
image_tag: [2.6-latest]
case_tag: [L0, L1, L2, RELEASE, RBAC]
steps:
- uses: actions/checkout@v7
- name: Set up Python 3.10
uses: actions/setup-python@v7
with:
python-version: '3.10'
cache: pip
- uses: actions/setup-go@v7
with:
go-version: ${{ env.go-version }}
cache: true
- name: Build
timeout-minutes: 5
shell: bash
run: |
yq -i '.log.level = "debug"' configs/backup.yaml
cat configs/backup.yaml || true
go get
go build
bash build_image.sh
- name: Install dependency
timeout-minutes: 5
working-directory: tests
shell: bash
run: |
pip install -r requirements.txt --trusted-host https://test.pypi.org
- name: Milvus deploy
timeout-minutes: 15
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.case_tag }} == "RELEASE" ]; then
mv docker-compose-tei.yml docker-compose.yml
fi
if [ ${{ matrix.case_tag }} == "RBAC" ]; then
mv docker-compose-rbac.yml docker-compose.yml
fi
tag=$(python ../../scripts/get_image_tag_by_short_name.py --tag ${{ matrix.image_tag }}) && echo $tag
yq -i ".services.standalone.image=\"milvusdb/milvus:${tag}\"" docker-compose.yml
cat custom_config.yaml || true
mkdir -p volumes/etcd volumes/minio volumes/milvus
sudo chmod -R 777 volumes
sudo docker compose -f docker-compose.yml up -d --wait
sudo docker compose -f docker-compose.yml ps -a
- name: Start backup server
timeout-minutes: 5
shell: bash
working-directory: build/deployment
run: |
sudo docker compose -f docker-compose.yml up -d --wait
sleep 10
sudo docker compose -f docker-compose.yml ps -a
- name: Run test
timeout-minutes: 120
shell: bash
working-directory: tests
run: |
if [ ${{ matrix.case_tag }} == "RBAC" ]; then
pytest -s -v --tags ${{ matrix.case_tag }}
else
pytest -s -v --tags ${{ matrix.case_tag }} -n 4 --tei_endpoint http://text-embeddings:80
fi
- name: Get Milvus status
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
sudo docker compose -f docker-compose.yml ps -a || true
- name: Export Milvus logs
if: ${{ always() }}
shell: bash
working-directory: deployment/${{ matrix.milvus_mode }}
run: |
if [ ${{ matrix.deploy_tools}} == "docker-compose" ]; then
bash ../../scripts/export_log_docker.sh logs
fi
- name: Export Backup Server logs
if: ${{ always() }}
shell: bash
working-directory: build/deployment
run: |
sudo docker compose -f docker-compose.yml logs > /tmp/ci_logs/backup_server.log 2>&1 || true
- name: Upload logs
if: ${{ ! success() }}
uses: actions/upload-artifact@v7
with:
name: api-test-logs-${{ matrix.deploy_tools }}-${{ matrix.milvus_mode }}-${{ matrix.image_tag }}-${{ matrix.case_tag }}
path: |
/tmp/ci_logs
deployment/${{ matrix.milvus_mode }}/logs
ci-summary:
name: CI summary
if: always()
needs:
- lint
- unit-test-go
- check-sample-configs
- test-backup-restore-cross-version
- test-backup-restore-after-upgrade
- test-backup-restore-cli
- test-backup-restore-secondary
- test-backup-restore-with-rbac-config
- test-backup-restore-api
runs-on: ubuntu-latest
steps:
- name: Check results
run: |
results=( ${{ join(needs.*.result, ' ') }} )
for r in "${results[@]}"; do
if [[ "$r" != "success" && "$r" != "skipped" ]]; then
echo "Job failed with status: $r"
exit 1
fi
done