conda-build: Build for Windows #11399
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build with conda | |
| on: | |
| push: | |
| branches: | |
| - master | |
| # For Pull-Requests, this runs the CI on merge commit | |
| # of HEAD with the target branch instead on HEAD, allowing | |
| # testing against potential new states which might have | |
| # been introduced in the target branch last commits. | |
| # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| run_on_arm_mac: | |
| description: 'Run on arm macos' | |
| type: boolean | |
| required: false | |
| default: false | |
| run_cpp_tests: | |
| description: 'Run C++ tests' | |
| type: boolean | |
| required: true | |
| default: true | |
| persistent_storage: | |
| description: "Run against what persistent storage type? (no is LMDB/default)" | |
| type: choice | |
| options: | |
| - 'no' | |
| - 'AWS_S3' | |
| - 'GCPXML' | |
| - 'AZURE' | |
| default: 'no' | |
| debug_enabled: | |
| type: boolean | |
| description: 'Run the build with debugging enabled' | |
| required: false | |
| default: false | |
| run_enable_logging: | |
| description: 'Enabled debug logging' | |
| type: boolean | |
| required: false | |
| default: false | |
| run_commandline: | |
| description: 'Run custom commandline before tests, Like: export ARCTICDB_STORAGE_AZURE=1; ....' | |
| type: string | |
| required: false | |
| default: "" | |
| run_custom_pytest_command: | |
| description: '*Run custom pytest command, instead of standard(Note: curdir is project root), or pass additional arguments to default command' | |
| type: string | |
| required: false | |
| default: "" | |
| jobs: | |
| linux: | |
| if: | | |
| always() && | |
| !cancelled() | |
| runs-on: ubuntu-22.04 | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| services: | |
| mongodb: | |
| image: mongo:4.4 | |
| steps: | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/[email protected] | |
| with: | |
| tool-cache: false | |
| large-packages: false # Time-consuming but doesn't save that much space (4GB) | |
| docker-images: false # We're using docker images we don't want to clear | |
| - uses: actions/[email protected] | |
| # DONT use recursive submodules checkout to simulate conda feedstock build | |
| # with: | |
| # submodules: recursive | |
| - name: Get number of CPU cores | |
| uses: SimenB/[email protected] | |
| id: cpu-cores | |
| - name: Install Conda environment from environment-dev.yml | |
| uses: mamba-org/[email protected] | |
| with: | |
| environment-file: environment-dev.yml | |
| init-shell: >- | |
| bash | |
| cache-environment: true | |
| post-cleanup: 'all' | |
| - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) | |
| shell: bash -l {0} | |
| run: | | |
| # Protocol buffers compilation require not using build isolation. | |
| # We should always retry due to instable nature of connections and environments | |
| python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| ARCTICDB_BUILD_CPP_TESTS: 1 | |
| - name: Build C++ Tests | |
| if: ${{ inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch' }} | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/linux-conda-release-build/ | |
| make -j ${{ steps.cpu-cores.outputs.count }} arcticdb_rapidcheck_tests | |
| make -j ${{ steps.cpu-cores.outputs.count }} test_unit_arcticdb | |
| - name: Run C++ Tests | |
| if: ${{ inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch' }} | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/linux-conda-release-build/ | |
| CTEST_OUTPUT_ON_FAILURE=1 make test | |
| # Note: mongo tests are skipped in the macos workflow | |
| - name: Install MongoDB | |
| uses: ./.github/actions/install_mongodb | |
| - name: Install npm # Linux github runner image does not come with npm | |
| uses: actions/[email protected] | |
| with: | |
| node-version: '24' | |
| - name: Install Azurite | |
| uses: nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 10 | |
| # We should always retry due to instable nature of connections and environments | |
| max_attempts: 3 | |
| command: npm install -g azurite | |
| - name: Check no arcticdb file depend on tests package | |
| shell: bash -l {0} | |
| run: | | |
| build_tooling/checks.sh | |
| - name: Set persistent storage variables | |
| # Should be executed for all persistent storages but not for LMDB | |
| if: ${{ env.real_tests_storage_type != 'no' }} | |
| uses: ./.github/actions/set_persistent_storage_env_vars | |
| with: | |
| aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" | |
| aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" | |
| gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" | |
| gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" | |
| azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE | |
| azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" | |
| persistent_storage: ${{ inputs.persistent_storage || 'no' }} | |
| - name: Set ArcticDB Debug Logging | |
| if: ${{ inputs.run_enable_logging }} | |
| uses: ./.github/actions/enable_logging | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ inputs.debug_enabled }} | |
| - name: Test with pytest | |
| shell: bash -l {0} | |
| run: | | |
| echo "Run commandline: $COMMANDLINE" | |
| eval "$COMMANDLINE" | |
| export ARCTICDB_RAND_SEED=$RANDOM | |
| export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 | |
| if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == *pytest* ]]; then | |
| command="python -m $ARCTICDB_PYTEST_ARGS" | |
| python -m pip --retries 3 --timeout 180 install pytest-repeat | |
| echo "Run custom pytest command: $command" | |
| eval "$command" | |
| else | |
| cd python | |
| python -m pytest --timeout=3600 -v -n logical --dist worksteal tests $ARCTICDB_PYTEST_ARGS | |
| fi | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| COMMANDLINE: ${{ inputs.run_commandline }} | |
| # Use the Mongo created in the service container above to test against | |
| CI_MONGO_HOST: mongodb | |
| ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }} | |
| NODE_OPTIONS: --openssl-legacy-provider | |
| macos: | |
| if: | | |
| always() && | |
| !cancelled() | |
| runs-on: macos-14 | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true | |
| steps: | |
| - uses: actions/[email protected] | |
| # DONT use recursive submodules checkout to simulate conda feedstock build | |
| # with: | |
| # submodules: recursive | |
| - name: Get number of CPU cores | |
| uses: SimenB/[email protected] | |
| id: cpu-cores | |
| - name: Install Conda environment from environment-dev.yml | |
| uses: mamba-org/[email protected] | |
| with: | |
| environment-file: environment-dev.yml | |
| environment-name: arcticdb | |
| init-shell: >- | |
| bash | |
| cache-environment: true | |
| post-cleanup: 'all' | |
| - name: Build ArcticDB with conda (ARCTICDB_USING_CONDA=1) | |
| shell: bash -l {0} | |
| run: | | |
| # Protocol buffers compilation require not using build isolation. | |
| python -m pip install --no-build-isolation --no-deps --retries 3 --timeout 400 -v -e . | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| ARCTICDB_BUILD_CPP_TESTS: 1 | |
| - name: Build C++ Tests | |
| if: ${{ inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch' }} | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/macos-conda-release-build/ | |
| make -j ${{ steps.cpu-cores.outputs.count }} arcticdb_rapidcheck_tests | |
| make -j ${{ steps.cpu-cores.outputs.count }} test_unit_arcticdb | |
| - name: Run C++ Tests | |
| if: ${{ inputs.run_cpp_tests == true|| github.event_name != 'workflow_dispatch' }} | |
| shell: bash -l {0} | |
| run: | | |
| cd cpp/out/macos-conda-release-build/ | |
| CTEST_OUTPUT_ON_FAILURE=1 make test | |
| - name: Install npm | |
| uses: actions/[email protected] | |
| with: | |
| node-version: '24' | |
| - name: Install Azurite | |
| uses: nick-fields/retry@v3 | |
| with: | |
| # We should always retry due to instable nature of connections and environments | |
| timeout_minutes: 10 | |
| max_attempts: 3 | |
| command: npm install -g azurite | |
| - name: Set persistent storage variables | |
| uses: ./.github/actions/set_persistent_storage_env_vars | |
| with: | |
| aws_access_key: "${{ secrets.AWS_S3_ACCESS_KEY }}" | |
| aws_secret_key: "${{ secrets.AWS_S3_SECRET_KEY }}" | |
| gcp_access_key: "${{ secrets.GCP_S3_ACCESS_KEY }}" | |
| gcp_secret_key: "${{ secrets.GCP_S3_SECRET_KEY }}" | |
| azure_container: "githubblob" # DEFAULT BUCKET FOR AZURE | |
| azure_connection_string: "${{ secrets.AZURE_CONNECTION_STRING }}" | |
| persistent_storage: ${{ inputs.persistent_storage || 'no' }} | |
| - name: Set ArcticDB Debug Logging | |
| if: ${{ inputs.run_enable_logging }} | |
| uses: ./.github/actions/enable_logging | |
| - name: Setup tmate session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: ${{ inputs.debug_enabled }} | |
| - name: Test with pytest | |
| shell: bash -l {0} | |
| run: | | |
| echo "Run commandline: $COMMANDLINE" | |
| eval "$COMMANDLINE" | |
| export ARCTICDB_RAND_SEED=$RANDOM | |
| export ARCTICDB_WARN_ON_WRITING_EMPTY_DATAFRAME=0 | |
| if [[ "$(echo "$ARCTICDB_PYTEST_ARGS" | xargs)" == pytest* ]]; then | |
| command="python -m $ARCTICDB_PYTEST_ARGS" | |
| echo "Run custom pytest command: $command" | |
| python -m pip install --retries 3 --timeout 240 pytest-repeat | |
| echo "Run custom pytest command: $ARCTICDB_PYTEST_ARGS" | |
| eval "$command" | |
| else | |
| cd python | |
| python -m pytest --timeout=3600 -v -n logical --dist worksteal tests $ARCTICDB_PYTEST_ARGS | |
| fi | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| COMMANDLINE: ${{ inputs.run_commandline }} | |
| ARCTICDB_PYTEST_ARGS: ${{ inputs.run_custom_pytest_command }} | |
| NODE_OPTIONS: --openssl-legacy-provider | |
| windows: | |
| runs-on: windows-latest | |
| env: | |
| ACTIONS_ALLOW_UNSECURE_COMMANDS: true | |
| SCCACHE_GHA_VERSION: ${{vars.SCCACHE_GHA_VERSION || 1}} | |
| steps: | |
| - uses: actions/[email protected] | |
| # DONT use recursive submodules checkout to simulate conda feedstock build | |
| # with: | |
| # submodules: recursive | |
| - name: Configure sccache | |
| uses: mozilla-actions/[email protected] | |
| with: | |
| sccache-version: ${{vars.SCCACHE_GHA_VERSION || 1}} | |
| - name: Install Conda environment from environment-dev.yml | |
| uses: mamba-org/[email protected] | |
| with: | |
| environment-file: environment-dev.yml | |
| init-shell: bash cmd.exe | |
| cache-environment: true | |
| post-cleanup: 'all' | |
| - name: Build ArcticDB C++ target (arcticdb_ext) for conda | |
| shell: cmd /C call {0} | |
| # Empty values for `CMAKE_GENERATOR_{PLATFORM,TOOLSET}` is necessary | |
| # for Ninja to be used as a generator with MSVC. | |
| run: | | |
| set CMAKE_GENERATOR_PLATFORM= | |
| set CMAKE_GENERATOR_TOOLSET= | |
| cd cpp | |
| cmake -DTEST=ON --preset windows-cl-conda-release | |
| cmake --build --preset windows-cl-conda-release --parallel | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| ARCTICDB_BUILD_CPP_TESTS: 1 | |
| ARCTIC_CMAKE_PRESET: windows-cl-conda-release | |
| - name: Build and Run C++ Tests | |
| shell: cmd /C call {0} | |
| if: false # ${{ inputs.run_cpp_tests == true || github.event_name != 'workflow_dispatch' }} | |
| # Empty values for `CMAKE_GENERATOR_{PLATFORM,TOOLSET}` is necessary | |
| # for Ninja to be used as a generator with MSVC. | |
| # Build both test targets in a single command to reuse artifacts from previous step | |
| run: | | |
| set CMAKE_GENERATOR_PLATFORM= | |
| set CMAKE_GENERATOR_TOOLSET= | |
| cd cpp/out/windows-cl-conda-release-build/ | |
| cmake --build . --target arcticdb_rapidcheck_tests test_unit_arcticdb --parallel | |
| ctest --output-on-failure | |
| env: | |
| CTEST_OUTPUT_ON_FAILURE: 1 | |
| ARCTICDB_USING_CONDA: 1 | |
| ARCTICDB_BUILD_CPP_TESTS: 1 | |
| ARCTIC_CMAKE_PRESET: windows-cl-conda-release | |
| - name: Build Python target for conda | |
| shell: cmd /C call {0} | |
| run: | | |
| python -m pip install --no-build-isolation --no-deps -v -e . | |
| env: | |
| ARCTICDB_USING_CONDA: 1 | |
| ARCTICDB_BUILD_CPP_TESTS: 1 | |
| ARCTIC_CMAKE_PRESET: windows-cl-conda-release | |
| - name: Install npm | |
| uses: actions/[email protected] | |
| with: | |
| node-version: '16' | |
| - name: Install azurite | |
| shell: bash -elo pipefail {0} | |
| run: | | |
| npm install -g azurite | |
| - name: Run Python Tests | |
| shell: bash -elo pipefail {0} | |
| run: | | |
| cd python | |
| export ARCTICDB_RAND_SEED=$RANDOM | |
| python -m pytest --timeout=3600 |