Continuous Benchmark Shard Transfer Speed #137
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Continuous Benchmark Shard Transfer Speed | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| dataset_name: | |
| description: 'Dataset name' | |
| default: 'h-and-m-2048-angular-filters' | |
| type: string | |
| qdrant_versions: | |
| description: 'Comma-separated versions to compare (ghcr/dev, docker/master, docker/v1.13.0). Can be just single version.' | |
| default: 'ghcr/dev,docker/master' | |
| type: string | |
| cluster_nodes: | |
| description: 'Number of cluster nodes' | |
| default: '2' | |
| type: string | |
| region: | |
| description: 'Hetzner region' | |
| default: 'fsn1' | |
| type: string | |
| server_type: | |
| description: 'Hetzner server type' | |
| default: 'ccx13' | |
| type: string | |
| client_type: | |
| description: 'Hetzner client type' | |
| default: 'cx33' | |
| type: string | |
| schedule: | |
| - cron: "0 4 * * *" # Daily at 4am UTC | |
| concurrency: | |
| group: hetzner-machines | |
| env: | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| # Defaults for push trigger (inputs.* is empty on push) | |
| SERVER_TYPE: ${{ inputs.server_type || 'ccx13' }} | |
| CLIENT_TYPE: ${{ inputs.client_type || 'cx33' }} | |
| REGION: ${{ inputs.region || 'fsn1' }} | |
| DATASET_NAME: ${{ inputs.dataset_name || 'h-and-m-2048-angular-filters' }} | |
| QDRANT_VERSIONS: ${{ inputs.qdrant_versions || 'ghcr/dev,docker/master' }} | |
| CLUSTER_NODES: ${{ inputs.cluster_nodes || '2' }} | |
| jobs: | |
| generateMatrix: | |
| name: Generate Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.generate.outputs.matrix }} | |
| node_names: ${{ steps.generate.outputs.node_names }} | |
| client_name: ${{ steps.generate.outputs.client_name }} | |
| steps: | |
| - name: Generate machine matrix | |
| id: generate | |
| run: | | |
| RUN_ID=${{ github.run_id }} | |
| python >> $GITHUB_OUTPUT << "EOF" | |
| import json | |
| import os | |
| NODES = int(os.environ["CLUSTER_NODES"]) | |
| RUN_ID = ${{ github.run_id }} | |
| machines = [] | |
| node_names = [] | |
| for i in range(NODES): | |
| machines.append({ | |
| "name": f"node-{i}", "suffix": f"node-{i}", "type": "SERVER_TYPE" | |
| }) | |
| node_names.append(f"transfer-bench-node-{i}-{RUN_ID}") | |
| machines.append({ | |
| "name": "client", "suffix": "client", "type": "CLIENT_TYPE" | |
| }) | |
| print(f"matrix={json.dumps({'machine': machines})}") | |
| print(f'node_names={" ".join(node_names)}') | |
| print(f"client_name=transfer-bench-client-{RUN_ID}") | |
| EOF | |
| setupCluster: | |
| name: Setup ${{ matrix.machine.name }} | |
| needs: generateMatrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: true | |
| matrix: ${{ fromJSON(needs.generateMatrix.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 # v0.10.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Setup CI tools | |
| run: bash -x tools/setup_ci.sh | |
| - name: Create machine | |
| uses: ./.github/workflows/actions/create-server-with-retry | |
| with: | |
| server_name: transfer-bench-${{ matrix.machine.suffix }}-${{ github.run_id }} | |
| server_type: ${{ matrix.machine.type == 'CLIENT_TYPE' && env.CLIENT_TYPE || env.SERVER_TYPE }} | |
| region: ${{ env.REGION }} | |
| max_retries: 2 | |
| runBenchmark: | |
| name: Run Transfer Benchmark | |
| needs: [generateMatrix, setupCluster] | |
| runs-on: ubuntu-latest | |
| container: alpine/ansible:2.18.1 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - uses: webfactory/ssh-agent@e83874834305fe9a4a2997156cb26c5de65a8555 # v0.10.0 | |
| with: | |
| ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} | |
| - name: Create inventory | |
| uses: ./.github/workflows/actions/create-inventory | |
| with: | |
| hcloud_token: ${{ secrets.HCLOUD_TOKEN }} | |
| server_names: ${{ needs.generateMatrix.outputs.node_names }} | |
| client_names: ${{ needs.generateMatrix.outputs.client_name }} | |
| db_host: ${{ secrets.POSTGRES_HOST }} | |
| - name: Run benchmarks for all versions | |
| run: | | |
| echo "$QDRANT_VERSIONS" | tr ',' '\n' | while read -r QDRANT_VERSION; do | |
| QDRANT_VERSION=$(echo "$QDRANT_VERSION" | xargs) | |
| [ -z "$QDRANT_VERSION" ] && continue | |
| echo "==========================================" | |
| echo "Benchmarking Qdrant version: $QDRANT_VERSION" | |
| echo "==========================================" | |
| case "${QDRANT_VERSION}" in | |
| docker/*) | |
| CONTAINER_REGISTRY="docker.io" | |
| VERSION=${QDRANT_VERSION#docker/} | |
| ;; | |
| ghcr/*) | |
| CONTAINER_REGISTRY="ghcr.io" | |
| VERSION=${QDRANT_VERSION#ghcr/} | |
| ;; | |
| *) | |
| echo "Error: unknown version ${QDRANT_VERSION}. Version name should start with 'docker/' or 'ghcr/'" | |
| exit 1 | |
| ;; | |
| esac | |
| ansible-playbook ansible/playbooks/playbook-transfer-speed.yml \ | |
| -i ansible/playbooks/inventory.ini \ | |
| --extra-vars "dataset_name=${DATASET_NAME} server_registry=${CONTAINER_REGISTRY} server_version=${VERSION}" | |
| done | |
| env: | |
| ANSIBLE_HOST_KEY_CHECKING: "False" | |
| ANSIBLE_SSH_ARGS: "-o ServerAliveInterval=30 -o ServerAliveCountMax=10 -o ControlMaster=no" | |
| ANSIBLE_STDOUT_CALLBACK: yaml | |
| QDRANT_VERSIONS: ${{ env.QDRANT_VERSIONS }} | |
| DATASET_NAME: ${{ env.DATASET_NAME }} | |
| cleanup: | |
| name: Cleanup Cluster | |
| needs: [generateMatrix, runBenchmark] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup CI tools | |
| run: bash -x tools/setup_ci.sh | |
| env: | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| - name: Delete machines | |
| env: | |
| NODE_NAMES: ${{ needs.generateMatrix.outputs.node_names }} | |
| CLIENT_NAME: ${{ needs.generateMatrix.outputs.client_name }} | |
| HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} | |
| run: echo $NODE_NAMES $CLIENT_NAME | xargs -P0 -n1 hcloud server delete |