diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index 9dd8df560..8e0767b9a 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -4,6 +4,9 @@ on: # Trigger the workflow on push or pull request, # but only for the main branch workflow_dispatch: + release: + types: + - created pull_request: branches: - main @@ -53,11 +56,11 @@ jobs: is_pull_request: ${{ github.event_name == 'pull_request' }} # ============================================================ - # Unified Jobs: Build NeuG + run C++, Python, and E2E tests + # Push / workflow_dispatch: build both x86_64 and arm64 # ============================================================ build_and_test_linux_x86_64: needs: format_check - if: ${{ (needs.format_check.result == 'success' || needs.format_check.result == 'skipped') && (github.event_name != 'push' || github.repository != 'alibaba/neug' || github.ref == 'refs/heads/main') }} + if: ${{ (needs.format_check.result == 'success' || needs.format_check.result == 'skipped') && (github.event_name != 'push' || github.repository != 'alibaba/neug' || github.ref == 'refs/heads/main') && contains(fromJSON('["push", "workflow_dispatch", "release"]'), github.event_name) }} uses: ./.github/workflows/build-and-test-common.yml with: runs-on: ${{ github.repository == 'alibaba/neug' && '["self-hosted", "daily", "linux", "x64"]' || '["ubuntu-22.04"]' }} @@ -70,7 +73,7 @@ jobs: build_and_test_linux_arm64: needs: format_check - if: ${{ (needs.format_check.result == 'success' || needs.format_check.result == 'skipped') && (github.event_name != 'push' || github.repository != 'alibaba/neug' || github.ref == 'refs/heads/main') }} + if: ${{ (needs.format_check.result == 'success' || needs.format_check.result == 'skipped') && (github.event_name != 'push' || github.repository != 'alibaba/neug' || github.ref == 'refs/heads/main') && contains(fromJSON('["push", "workflow_dispatch", "release"]'), github.event_name) }} uses: ./.github/workflows/build-and-test-common.yml with: runs-on: ${{ github.repository == 'alibaba/neug' && '["self-hosted", "daily", "linux", "arm64"]' || '["ubuntu-22.04-arm"]' }} @@ -80,3 +83,22 @@ jobs: enable-coverage: true enable-gopt-test: true event-name: ${{ github.event_name }} + + # ============================================================ + # Pull Request: run a single random architecture + # - self-hosted: omit arch label to let the runner pool pick randomly + # - GitHub-hosted: fallback to x86_64 (ubuntu-22.04) + # ============================================================ + build_and_test_linux_pr: + name: Build and test Linux (PR random arch) + needs: format_check + if: ${{ (needs.format_check.result == 'success' || needs.format_check.result == 'skipped') && github.event_name == 'pull_request' }} + uses: ./.github/workflows/build-and-test-common.yml + with: + runs-on: ${{ github.repository == 'alibaba/neug' && '["self-hosted", "daily", "linux"]' || '["ubuntu-22.04"]' }} + container-json: '{"image":"neug-registry.cn-hongkong.cr.aliyuncs.com/neug/neug-dev:v0.1.3"}' + os-type: linux + enable-java-test: true + enable-coverage: true + enable-gopt-test: true + event-name: ${{ github.event_name }} diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml index 4fb17260d..b9b2c6f78 100644 --- a/.github/workflows/OSX.yml +++ b/.github/workflows/OSX.yml @@ -2,22 +2,9 @@ name: NeuG Test (macOS arm64) on: workflow_dispatch: - pull_request: - paths: - - 'cmake/**' - - 'src/**' - - 'bin/**' - - 'proto/**' - - 'third_party/**' - - 'tools/python_bind/**' - - 'tools/utils/**' - - 'tests/**' - - '.github/workflows/OSX.yml' - - '.github/workflows/build-and-test-common.yml' - - 'scripts/install_deps.sh' - - 'include/**' - - 'CMakeLists.txt' - - 'doc/source/tutorials/**' + release: + types: + - created push: paths: - 'cmake/**' @@ -42,8 +29,8 @@ concurrency: jobs: format_check: if: | - github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || + github.event_name == 'release' || (github.event_name == 'push' && (github.repository != 'alibaba/neug' || github.ref == 'refs/heads/main')) uses: ./.github/workflows/format-check.yml with: @@ -55,7 +42,7 @@ jobs: # ============================================================ build_and_test_macos_arm64: needs: format_check - if: ${{ (needs.format_check.result == 'success' || needs.format_check.result == 'skipped') && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && (github.repository != 'alibaba/neug' || github.ref == 'refs/heads/main'))) }} + if: ${{ (needs.format_check.result == 'success' || needs.format_check.result == 'skipped') && (github.event_name == 'workflow_dispatch' || github.event_name == 'release' || (github.event_name == 'push' && (github.repository != 'alibaba/neug' || github.ref == 'refs/heads/main'))) }} uses: ./.github/workflows/build-and-test-common.yml with: runs-on: '["macos-14"]' diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index 751cadb4e..000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: NeuG Benchmark - -on: - # Trigger the workflow on push or pull request, - # but only for the main branch - workflow_dispatch: - schedule: - # The notifications for scheduled workflows are sent to the user who - # last modified the cron syntax in the workflow file. - # Trigger the workflow at 22:00(CST) every day. - - cron: '0 14 * * *' - push: - tags: - - "v*" - pull_request: - branches: - - main - paths: - - '.github/workflows/benchmark.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - lsqb-benchmark: - runs-on: ubuntu-22.04 - container: - image: neug-registry.cn-hongkong.cr.aliyuncs.com/neug/neug-dev:v0.1.3 - # options: --user root - steps: - - name: checkout - id: checkout - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Cache CCache - if: false - uses: actions/cache@v4 - with: - path: ~/.cache - # use timestamp as cache key to avoid cache miss - key: ${{ runner.os }}-ccache-${{ github.run_id }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-ccache-${{ github.run_id }}- - ${{ runner.os }}-ccache- - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.13' - - - name: Build NeuG - run: | - . /home/neug/.neug_env - cd ${GITHUB_WORKSPACE}/ - export BUILD_TYPE=RELEASE - export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) - export BUILD_EXECUTABLES=ON - export BUILD_HTTP_SERVER=ON - export BUILD_TEST=OFF - export USE_NINJA=OFF - # export CC="ccache gcc" - # export CXX="ccache g++" - cd tools/python_bind - make clean - make requirements - make build - - - name: Bulk Loader Test - env: - GS_TEST_DIR: ${{ github.workspace }}/gstest - run: | - git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR} - rm -rf /tmp/modern_graph && rm -rf /tmp/lsqb && rm -rf /tmp/ldbc && rm -rf /tmp/tinysnb - cd ${GITHUB_WORKSPACE}/tools/python_bind/ - export FLEX_DATA_DIR=${GS_TEST_DIR}/neug/lsqb-sf01 - GLOG_v=10 ${GITHUB_WORKSPACE}/build/tools/utils/bulk_loader -g ${GS_TEST_DIR}/neug/lsqb-sf01/graph.yaml -l ${GS_TEST_DIR}/neug/lsqb-sf01/import.yaml -d /tmp/lsqb - - - name: Run E2E LSQB Benchmark - run: | - cd ${GITHUB_WORKSPACE}/tests/e2e - # run lsqb benchmark - python3 -m pytest -m neug_benchmark -sv --db_dir=/tmp/lsqb/ --query_dir=./queries/lsqb/ --dataset lsqb --iterations 1 --rounds 1 --read_only --benchmark-save=benchmark-lsqb - - notify-benchmark-result: - needs: [lsqb-benchmark] - if: always() && github.repository == 'alibaba/neug' - runs-on: ubuntu-latest - steps: - - name: Send DingTalk notification - env: - DINGTALK_TOKEN: ${{ secrets.DINGTALK_TOKEN }} - run: | - if [ -z "$DINGTALK_TOKEN" ]; then - echo "DINGTALK_TOKEN not set, skipping notification." - exit 0 - fi - WEBHOOK_URL="https://oapi.dingtalk.com/robot/send?access_token=${DINGTALK_TOKEN}" - STATUS="${{ needs.lsqb-benchmark.result }}" - if [[ "$STATUS" == "success" ]]; then - TITLE="[Nightly Report] NeuG Benchmark Passed" - else - TITLE="[Nightly Report] NeuG Benchmark Failed" - fi - curl -s -X POST "$WEBHOOK_URL" \ - -H 'Content-Type: application/json' \ - -d "{ - \"msgtype\": \"markdown\", - \"markdown\": { - \"title\": \"$TITLE\", - \"text\": \"## $TITLE\n\n- **lsqb-benchmark**: ${STATUS}\n- **Repo**: ${{ github.repository }}\n- **Branch**: ${{ github.ref_name }}\n- **Commit**: ${{ github.sha }}\n- [View Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\" - } - }" diff --git a/.github/workflows/nightly-npm-package-test.yml b/.github/workflows/npm-package-test.yml similarity index 95% rename from .github/workflows/nightly-npm-package-test.yml rename to .github/workflows/npm-package-test.yml index 5d6a9a735..63f3d8ff4 100644 --- a/.github/workflows/nightly-npm-package-test.yml +++ b/.github/workflows/npm-package-test.yml @@ -1,4 +1,4 @@ -name: Nightly NPM Package Test +name: NPM Package Test on: schedule: @@ -9,14 +9,14 @@ on: branches: - main paths: - - '.github/workflows/nightly-npm-package-test.yml' + - '.github/workflows/npm-package-test.yml' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true jobs: - nightly-test: + package-test: runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/nightly-pypi-package-test.yml b/.github/workflows/pypi-package-test.yml similarity index 93% rename from .github/workflows/nightly-pypi-package-test.yml rename to .github/workflows/pypi-package-test.yml index 4ce385963..3b0ae8e20 100644 --- a/.github/workflows/nightly-pypi-package-test.yml +++ b/.github/workflows/pypi-package-test.yml @@ -1,4 +1,4 @@ -name: Nightly PyPI Package Test +name: PyPI Package Test on: schedule: @@ -9,14 +9,14 @@ on: branches: - main paths: - - '.github/workflows/nightly-pypi-package-test.yml' + - '.github/workflows/pypi-package-test.yml' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true jobs: - nightly-test: + package-test: runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/release-npm.yml similarity index 99% rename from .github/workflows/npm-publish.yml rename to .github/workflows/release-npm.yml index 3cdf2429a..23b0bbcff 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/release-npm.yml @@ -1,4 +1,4 @@ -name: NPM Publish +name: Release NPM Packages # Published GitHub releases drive npm publishing. on: diff --git a/.github/workflows/build-wheel.yml b/.github/workflows/release-wheel.yml similarity index 64% rename from .github/workflows/build-wheel.yml rename to .github/workflows/release-wheel.yml index 17ea15688..fc873b2fd 100644 --- a/.github/workflows/build-wheel.yml +++ b/.github/workflows/release-wheel.yml @@ -1,8 +1,6 @@ -name: Build Wheels +name: Release Wheels on: - # Trigger the workflow on push or pull request, - # but only for the main branch workflow_dispatch: inputs: python_versions: @@ -12,30 +10,16 @@ on: release: types: - created - push: - branches: - - main - tags: - - 'v*' - paths: &wheel_paths - - 'cmake/**' - - 'src/**' - - 'include/**' - - 'proto/**' - - 'third_party/**' - - 'tools/python_bind/**' - - '!tools/python_bind/tests/**' - - '!tools/python_bind/example/**' - - 'tools/shell/**' - - '.github/workflows/build-wheel.yml' - - '.github/workflows/wheels-common.yml' - - 'scripts/install_deps.sh' - - 'CMakeLists.txt' - - 'NEUG_VERSION' pull_request: branches: - main - paths: *wheel_paths + paths: + - '.github/workflows/release-wheel.yml' + - '.github/workflows/wheels-common.yml' + - 'tools/python_bind/setup.py' + - 'tools/python_bind/pyproject.toml' + - 'tools/python_bind/Makefile' + - 'tools/python_bind/README.md' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} @@ -43,7 +27,7 @@ concurrency: jobs: format_check: - if: github.event_name == 'pull_request' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' || github.event_name == 'release' uses: ./.github/workflows/format-check.yml with: is_pull_request: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/weekly-build-image.yml similarity index 82% rename from .github/workflows/build-image.yml rename to .github/workflows/weekly-build-image.yml index d5fff9fbe..27be55b6a 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/weekly-build-image.yml @@ -1,4 +1,4 @@ -name: Build NeuG image +name: Weekly Build NeuG image on: # Trigger the workflow on push or pull request, @@ -7,8 +7,8 @@ on: schedule: # The notifications for scheduled workflows are sent to the user who # last modified the cron syntax in the workflow file. - # Trigger the workflow at 22:00(CST) every day. - - cron: '0 14 * * *' + # Trigger the workflow at 22:00(CST) every Monday. + - cron: '0 14 * * 1' push: tags: - "v*" @@ -16,7 +16,7 @@ on: branches: - main paths: - - '.github/workflows/build-image.yml' + - '.github/workflows/weekly-build-image.yml' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} @@ -43,4 +43,4 @@ jobs: - name: Remove Image run: | ARCH=$(uname -m) - docker rmi neug/neug:${ARCH} \ No newline at end of file + docker rmi neug/neug:${ARCH} diff --git a/.github/workflows/build-nightly-wheel.yml b/.github/workflows/weekly-build-wheel.yml similarity index 87% rename from .github/workflows/build-nightly-wheel.yml rename to .github/workflows/weekly-build-wheel.yml index 6b30110c6..922cd3a0f 100644 --- a/.github/workflows/build-nightly-wheel.yml +++ b/.github/workflows/weekly-build-wheel.yml @@ -1,4 +1,4 @@ -name: Build Nightly Wheels +name: Build Weekly Wheels on: # Trigger the workflow on push or pull request, @@ -10,13 +10,13 @@ on: required: false default: '' schedule: - # Trigger the workflow at 22:00(CST) every day (14:00 UTC) - - cron: '0 14 * * *' + # Trigger the workflow at 22:00(CST) every Monday (14:00 UTC) + - cron: '0 14 * * 1' pull_request: branches: - main paths: - - '.github/workflows/build-nightly-wheel.yml' + - '.github/workflows/weekly-build-wheel.yml' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} @@ -41,7 +41,7 @@ jobs: # ============================================================ # Notify: Send results to DingTalk # ============================================================ - notify-nightly-wheel-result: + notify-weekly-wheel-result: needs: [build_wheels] if: always() && github.repository == 'alibaba/neug' runs-on: ubuntu-latest @@ -57,9 +57,9 @@ jobs: WEBHOOK_URL="https://oapi.dingtalk.com/robot/send?access_token=${DINGTALK_TOKEN}" STATUS="${{ needs.build_wheels.result }}" if [[ "$STATUS" == "success" ]]; then - TITLE="[Nightly Report] Build Nightly Wheels Passed" + TITLE="[Weekly Report] Build Weekly Wheels Passed" else - TITLE="[Nightly Report] Build Nightly Wheels Failed" + TITLE="[Weekly Report] Build Weekly Wheels Failed" fi curl -s -X POST "$WEBHOOK_URL" \ -H 'Content-Type: application/json' \ diff --git a/.github/workflows/ldbc-nightly-benchmark.yml b/.github/workflows/weekly-ldbc-benchmark.yml similarity index 96% rename from .github/workflows/ldbc-nightly-benchmark.yml rename to .github/workflows/weekly-ldbc-benchmark.yml index a72757b5e..f6d426ea2 100644 --- a/.github/workflows/ldbc-nightly-benchmark.yml +++ b/.github/workflows/weekly-ldbc-benchmark.yml @@ -1,10 +1,9 @@ -name: LDBC SNB Nightly Benchmark +name: LDBC SNB Weekly Benchmark on: schedule: - # UTC+8 03:00 Wednesday = UTC 19:00 Tuesday - # UTC+8 03:00 Sunday = UTC 19:00 Saturday - - cron: "0 19 * * 2,6" + # UTC+8 03:00 Sunday = UTC 19:00 Saturday + - cron: "0 19 * * 6" workflow_dispatch: env: diff --git a/.github/workflows/memory-leak-nightly.yml b/.github/workflows/weekly-memory-leak-test.yml similarity index 93% rename from .github/workflows/memory-leak-nightly.yml rename to .github/workflows/weekly-memory-leak-test.yml index 8a6551978..4e3e5134b 100644 --- a/.github/workflows/memory-leak-nightly.yml +++ b/.github/workflows/weekly-memory-leak-test.yml @@ -1,4 +1,4 @@ -name: Memory Leak Nightly +name: Memory Leak Weekly on: workflow_dispatch: @@ -16,8 +16,8 @@ on: required: false default: '2000' schedule: - # UTC 19:00 = UTC+8 03:00 every day - - cron: '0 19 * * *' + # UTC 19:00 Saturday = UTC+8 03:00 Sunday + - cron: '0 19 * * 6' concurrency: group: ${{ github.workflow }} @@ -191,7 +191,7 @@ jobs: - name: Summarize verdicts if: always() run: | - echo '## Memory Leak Nightly Summary' >> "$GITHUB_STEP_SUMMARY" + echo '## Memory Leak Weekly Summary' >> "$GITHUB_STEP_SUMMARY" echo '' >> "$GITHUB_STEP_SUMMARY" if [ -f "$LOG_DIR/test_node_create.log" ]; then echo '### gtest test_node_create' >> "$GITHUB_STEP_SUMMARY" @@ -216,13 +216,13 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: memory-leak-nightly-${{ github.run_id }} + name: weekly-memory-leak-test-${{ github.run_id }} path: ${{ env.LOG_DIR }} if-no-files-found: warn retention-days: 14 # ============================================================ - # Job: send DingTalk notification for the nightly memory-leak run. + # Job: send DingTalk notification for the weekly memory-leak run. # Mirrors benchmark.yml's notify-benchmark-result style. # Only runs on the canonical alibaba/neug repo to avoid fork noise. # ============================================================ @@ -242,9 +242,9 @@ jobs: WEBHOOK_URL="https://oapi.dingtalk.com/robot/send?access_token=${DINGTALK_TOKEN}" STATUS="${{ needs.memory-leak.result }}" if [[ "$STATUS" == "success" ]]; then - TITLE="[Nightly Report] NeuG Memory Leak Passed" + TITLE="[Weekly Report] NeuG Memory Leak Passed" else - TITLE="[Nightly Report] NeuG Memory Leak Failed" + TITLE="[Weekly Report] NeuG Memory Leak Failed" fi curl -s -X POST "$WEBHOOK_URL" \ -H 'Content-Type: application/json' \ @@ -252,6 +252,6 @@ jobs: \"msgtype\": \"markdown\", \"markdown\": { \"title\": \"$TITLE\", - \"text\": \"## $TITLE\n\n- **memory-leak**: ${STATUS}\n- **Repo**: ${{ github.repository }}\n- **Branch**: ${{ github.ref_name }}\n- **Commit**: ${{ github.sha }}\n- [View Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\" + \"text\": \"## $TITLE\\n\\n- **memory-leak**: ${STATUS}\\n- **Repo**: ${{ github.repository }}\\n- **Branch**: ${{ github.ref_name }}\\n- **Commit**: ${{ github.sha }}\\n- [View Run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})\" } }" diff --git a/README.md b/README.md index dc3944577..97e58c14f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@
A Graph Database for HTAP Workloads

NeuG Test (Linux) - NeuG Wheel Packaging + NeuG Wheel Packaging NeuG Documentation Coverage Discord