feature/ml-vectors: Initial sparse vector parsing and print #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test & Coverage | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup CMake 4.x | |
| uses: jwlawson/actions-setup-cmake@v2 | |
| with: | |
| cmake-version: '4.2.2' | |
| - name: Verify CMake version | |
| run: cmake --version | |
| - name: Install Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 17 | |
| - name: Install Spark | |
| run: | | |
| curl -fL https://dlcdn.apache.org/spark/spark-4.0.2/spark-4.0.2-bin-hadoop3.tgz -o spark.tgz | |
| tar -xzf spark.tgz | |
| mv spark-4.0.2-bin-hadoop3 $HOME/spark | |
| - name: Start Spark Connect Server | |
| run: | | |
| $HOME/spark/sbin/start-connect-server.sh \ | |
| --packages "org.apache.spark:spark-connect_2.13:4.0.2,io.delta:delta-spark_2.13:4.0.0,io.graphframes:graphframes-spark4_2.13:0.10.0,io.graphframes:graphframes-connect-spark4_2.13:0.10.0" \ | |
| --conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect" \ | |
| --conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError" \ | |
| --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ | |
| --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ | |
| --conf "spark.driver.memory=4g" \ | |
| --conf "spark.executor.memory=4g" \ | |
| --conf "spark.memory.fraction=0.8" \ | |
| --conf "spark.memory.storageFraction=0.3" \ | |
| --conf "spark.sql.shuffle.partitions=8" \ | |
| --conf "spark.default.parallelism=8" \ | |
| --conf "spark.driver.maxResultSize=2g" | |
| # wait for server | |
| sleep 15 | |
| - name: Install dependencies | |
| run: | | |
| cd scripts | |
| chmod +x install-deps.sh | |
| ./install-deps.sh | |
| - name: Install gcovr | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y gcovr | |
| - name: Verify Gcovr version | |
| run: gcovr --version | |
| - name: Configure | |
| run: cmake --preset pr_pipeline | |
| - name: Build | |
| run: cmake --build --preset pr_pipeline | |
| - name: Test | |
| run: | | |
| export SPARK_REMOTE=sc://localhost | |
| ctest --preset test_pr_pipeline_spark_coverage | |
| - name: Generate Coverage (XML & HTML) | |
| run: | | |
| gcovr -r src \ | |
| --object-directory build \ | |
| --exclude '.*\.pb\.cc' \ | |
| --exclude '.*\.grpc\.pb\.cc' \ | |
| --exclude '.*\.h' \ | |
| --xml-pretty -o coverage.xml \ | |
| --html-details coverage.html \ | |
| --fail-under-line 70 \ | |
| --print-summary \ | |
| --gcov-ignore-errors=no_working_dir_found | |
| - name: Upload Coverage XML | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-xml | |
| path: coverage.xml | |
| - name: Upload Coverage HTML | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-html | |
| path: coverage.html |