feat: Add executor pool size to S3 configs #41276

Workflow file for this run

.github/workflows/scheduled.yml at 56f6314

	# Copyright (c) Facebook, Inc. and its affiliates.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: Fuzzer Jobs

	on:
	pull_request:
	paths:
	- velox/**
	- '!velox/docs/**'
	- CMakeLists.txt
	- CMake/**
	- scripts/setup-ubuntu.sh
	- scripts/setup-common.sh
	- scripts/setup-versions.sh
	- scripts/setup-helper-functions.sh
	- .github/workflows/scheduled.yml
	- pyproject.toml

	push:
	branches:
	- main
	paths:
	- velox/**
	- '!velox/docs/**'
	- CMakeLists.txt
	- CMake/**
	- scripts/setup-ubuntu.sh
	- scripts/setup-common.sh
	- scripts/setup-versions.sh
	- scripts/setup-helper-functions.sh
	- .github/workflows/scheduled.yml
	- pyproject.toml

	schedule:
	- cron: 0 3 * * *

	workflow_dispatch:
	inputs:
	ref:
	description: Ref to checkout out
	default: main
	numThreads:
	description: Number of threads
	default: 16
	maxHighMemJobs:
	description: Number of high memory jobs
	default: 8
	maxLinkJobs:
	description: Maximum number of link jobs
	default: 4
	extraCMakeFlags:
	description: Additional CMake flags
	default: ''
	duration:
	description: Duration of fuzzer run in seconds
	default: 1800

	defaults:
	run:
	shell: bash

	permissions:
	contents: read

	concurrency:
	# This will not cancel fuzzer runs on main (regardless of which trigger)
	# by making the commit sha part of the group but will use the branch
	# name in PRs to cancel on going runs on a new commit.
	group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.head_ref \|\| github.sha }}
	cancel-in-progress: true

	env:
	# Per-instance fuzzer duration. With 4 parallel instances at 300s each,
	# total fuzzer coverage is 1200s (20 min) vs the previous 900s (15 min).
	DURATION: ${{ inputs.duration \|\| ( github.event_name != 'schedule' && 300 \|\| 600 )}}
	NUM_FUZZER_INSTANCES: 4
	# minimize artifact duration for PRs, keep them a bit longer for nightly runs
	RETENTION: ${{ github.event_name == 'pull_request' && 1 \|\| 3 }}

	jobs:
	compile:
	name: Build
	# prevent errors when forks ff their main branch
	if: ${{ github.repository == 'facebookincubator/velox' }}
	runs-on: 32-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	timeout-minutes: 120
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache
	NUM_THREADS: ${{ inputs.numThreads \|\| 32 }}
	MAX_HIGH_MEM_JOBS: ${{ inputs.maxHighMemJobs \|\| 8 }}
	MAX_LINK_JOBS: ${{ inputs.maxLinkJobs \|\| 6 }}
	SKBUILD_BUILD_DIR: _build/debug
	PYVELOX_LEGACY_ONLY: 'ON'
	CMAKE_POLICY_VERSION_MINIMUM: "3.5"

	defaults:
	run:
	shell: bash
	working-directory: velox
	outputs:
	presto_bias: ${{ steps.sig-check.outputs.presto_functions }}
	presto_error: ${{ steps.sig-check.outputs.presto_error }}
	spark_bias: ${{ steps.sig-check.outputs.spark_functions }}
	spark_error: ${{ steps.sig-check.outputs.spark_error }}
	presto_aggregate_bias: ${{ steps.sig-check.outputs.presto_aggregate_functions }}
	presto_aggregate_error: ${{ steps.sig-check.outputs.presto_aggregate_error }}
	signatures_checked: ${{ steps.sig-paths.outputs.signatures }}

	steps:

	- name: Get merge base with main
	if: ${{ github.event_name != 'schedule' }}
	working-directory: ${{ github.workspace }}
	env:
	GH_TOKEN: ${{ github.token }}
	INPUT_REF: ${{ inputs.ref \|\| 'main' }}
	BASE_REF: ${{ github.base_ref }}
	HEAD_REF: ${{ github.head_ref }}
	PR_OWNER: ${{ github.event.pull_request.head.repo.owner.login }}
	id: get-merge-base
	run: \|
	if [ '${{ github.event_name == 'push' }}' == "true" ]; then
	# get the parent commit of the current one to get the relevant function signatures
	head_main=$(gh api -q '.parents.[0].sha' '/repos/facebookincubator/velox/commits/${{ github.sha }}')
	elif [ '${{ github.event_name == 'pull_request' }}' == "true" ]; then
	# Get the merge base from the api
	head_main=$(gh api -q '.merge_base_commit.sha' \
	/repos/facebookincubator/velox/compare/facebookincubator:$BASE_REF...$PR_OWNER:$HEAD_REF \
	)
	else
	head_main=$INPUT_REF
	fi
	echo "Determined SHA: $head_main"
	echo "head_main=$head_main" >> $GITHUB_OUTPUT

	- name: Get Function Signature Stash
	uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
	id: get-sig
	with:
	path: /tmp/signatures
	key: function-signatures-${{ steps.get-merge-base.outputs.head_main \|\| github.sha }}

	- name: Restore ccache
	uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
	with:
	path: ${{ env.CCACHE_DIR }}
	key: ccache-fuzzer-centos

	- name: Fix git permissions
	working-directory: ${{ github.workspace }}
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: \|
	git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox
	git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox_main

	- name: Ensure Stash Dirs Exists
	working-directory: ${{ github.workspace }}
	run: \|
	mkdir -p '$CCACHE_DIR'
	mkdir -p /tmp/signatures

	- name: Check for signature-relevant changes
	uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
	if: github.event_name == 'pull_request'
	id: sig-paths
	with:
	filters: \|
	signatures:
	- 'velox/functions/prestosql/**'
	- 'velox/functions/sparksql/**'
	- 'velox/functions/lib/**'
	- 'velox/expression/**'
	- 'velox/exec/Aggregate.h'
	- 'velox/exec/Aggregate.cpp'
	- 'velox/type/**'
	- 'velox/python/**'
	- 'pyproject.toml'

	- name: Checkout Main
	if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' && (github.event_name != 'pull_request' \|\| steps.sig-paths.outputs.signatures == 'true') }}
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	ref: ${{ steps.get-merge-base.outputs.head_main \|\| 'main' }}
	path: velox_main
	persist-credentials: false

	- name: Build PyVelox
	if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' && (github.event_name != 'pull_request' \|\| steps.sig-paths.outputs.signatures == 'true') }}
	working-directory: velox_main
	run: \|
	make python-build

	- name: Create Baseline Signatures
	if: ${{ github.event_name != 'schedule' && steps.get-sig.outputs.stash-hit != 'true' && (github.event_name != 'pull_request' \|\| steps.sig-paths.outputs.signatures == 'true') }}
	working-directory: velox_main
	run: \|
	# TODO convert to uv with script defined deps after 14046 is merged
	source .venv/bin/activate
	PIP=$(command -v uv > /dev/null 2>&1 && echo "uv pip" \|\| echo "python3 -m pip")
	$PIP install deepdiff
	python3 scripts/ci/signature.py export --spark /tmp/signatures/spark_signatures_main.json
	python3 scripts/ci/signature.py export --presto /tmp/signatures/presto_signatures_main.json
	python3 scripts/ci/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_main.json

	- name: Save Function Signature Stash
	if: ${{ github.event_name == 'pull_request' && steps.get-sig.outputs.stash-hit != 'true' && steps.sig-paths.outputs.signatures == 'true' }}
	uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
	with:
	path: /tmp/signatures
	key: function-signatures-${{ steps.get-merge-base.outputs.head_main }}

	- name: Checkout Contender
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Zero Ccache Statistics
	run: \|
	ccache -sz

	- name: Build Fuzzer Targets
	env:
	EXTRA_CMAKE_FLAGS: >
	-DVELOX_ENABLE_ARROW=ON
	-DVELOX_ENABLE_GEO=ON
	-DVELOX_MONO_LIBRARY=ON
	-DVELOX_BUILD_SHARED=ON
	-DVELOX_BUILD_PYTHON_PACKAGE=ON
	-DVELOX_PYTHON_LEGACY_ONLY=ON
	${{ inputs.extraCMakeFlags }}
	run: \|
	make python-venv
	source .venv/bin/activate
	make cmake BUILD_DIR=debug BUILD_TYPE=Debug
	cmake --build _build/debug -j ${NUM_THREADS} --target \
	velox_expression_fuzzer_test \
	spark_expression_fuzzer_test \
	spark_aggregation_fuzzer_test \
	velox_aggregation_fuzzer_test \
	velox_join_fuzzer \
	velox_exchange_fuzzer \
	velox_window_fuzzer_test \
	velox_cache_fuzzer \
	velox_table_evolution_fuzzer_test \
	velox_memory_arbitration_fuzzer \
	velox_row_number_fuzzer \
	velox_topn_row_number_fuzzer \
	velox_writer_fuzzer_test \
	velox_spatial_join_fuzzer

	- name: Ccache after
	run: ccache -s

	- name: Save ccache
	# see https://github.com/actions/upload-artifact/issues/543
	continue-on-error: true
	if: ${{ github.event_name != 'schedule' }}
	uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
	with:
	path: ${{ env.CCACHE_DIR }}
	key: ccache-fuzzer-centos

	- name: Build PyVelox
	if: ${{ github.event_name != 'schedule' && (github.event_name != 'pull_request' \|\| steps.sig-paths.outputs.signatures == 'true') }}
	run: \|
	make python-build

	- name: Create and test new function signatures
	if: ${{ github.event_name != 'schedule' && (github.event_name != 'pull_request' \|\| steps.sig-paths.outputs.signatures == 'true') }}
	id: sig-check
	run: \|
	source .venv/bin/activate
	PIP=$(command -v uv > /dev/null 2>&1 && echo "uv pip" \|\| echo "python3 -m pip")
	$PIP install deepdiff
	python3 scripts/ci/signature.py gh_bias_check presto spark
	python3 scripts/ci/signature.py export_aggregates --presto /tmp/signatures/presto_aggregate_signatures_contender.json
	python3 scripts/ci/signature.py bias_aggregates /tmp/signatures/presto_aggregate_signatures_main.json \
	/tmp/signatures/presto_aggregate_signatures_contender.json /tmp/signatures/presto_aggregate_bias_functions \
	/tmp/signatures/presto_aggregate_errors

	- name: Upload Signature Artifacts
	if: ${{ github.event_name != 'schedule' && (github.event_name != 'pull_request' \|\| steps.sig-paths.outputs.signatures == 'true') }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: signatures
	path: /tmp/signatures
	retention-days: ${{ env.RETENTION }}

	- name: Prepare signatures
	working-directory: /tmp/signatures
	if: ${{ github.event_name == 'push' }}
	run: \|
	# Remove irrelevant artifacts
	rm -f *_bias_functions
	rm -f *_signatures_main.json
	# Rename signature files as 'main' files
	for f in *_signatures_contender.json; do
	mv "$f" "${f/_contender.json/_main.json}"
	done

	- name: Save Function Signature Stash
	if: ${{ github.event_name == 'push' }}
	uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
	with:
	path: /tmp/signatures
	key: function-signatures-${{ github.sha }}

	- name: Upload fuzzer scripts
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: fuzzer-scripts
	path: velox/scripts/ci/run-fuzzer-parallel.sh
	retention-days: ${{ env.RETENTION }}

	- name: Upload presto fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto
	path: velox/_build/debug/velox/expression/fuzzer/velox_expression_fuzzer_test
	retention-days: ${{ env.RETENTION }}

	- name: Upload spark expression fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: spark_expression_fuzzer
	path: velox/_build/debug/velox/expression/fuzzer/spark_expression_fuzzer_test
	retention-days: ${{ env.RETENTION }}

	- name: Upload spark aggregation fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: spark_aggregation_fuzzer
	path: velox/_build/debug/velox/functions/sparksql/fuzzer/spark_aggregation_fuzzer_test
	retention-days: ${{ env.RETENTION }}

	- name: Upload aggregation fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: aggregation
	path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_aggregation_fuzzer_test
	retention-days: ${{ env.RETENTION }}

	- name: Upload join fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: join
	path: velox/_build/debug/velox/exec/fuzzer/velox_join_fuzzer
	retention-days: ${{ env.RETENTION }}

	- name: Upload exchange fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: exchange
	path: velox/_build/debug//velox/exec/fuzzer/velox_exchange_fuzzer
	retention-days: ${{ env.RETENTION }}

	- name: Upload window fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: window
	path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_window_fuzzer_test
	retention-days: ${{ env.RETENTION }}

	- name: Upload cache fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: cache_fuzzer
	path: velox/_build/debug/velox/exec/fuzzer/velox_cache_fuzzer
	retention-days: ${{ env.RETENTION }}

	- name: Upload table evolution fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: table_evolution_fuzzer
	path: velox/_build/debug/velox/exec/tests/velox_table_evolution_fuzzer_test
	retention-days: ${{ env.RETENTION }}

	- name: Upload memory arbitration fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: memory_arbitration_fuzzer
	path: velox/_build/debug/velox/exec/fuzzer/velox_memory_arbitration_fuzzer
	retention-days: ${{ env.RETENTION }}

	- name: Upload row number fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: row_number
	path: velox/_build/debug//velox/exec/fuzzer/velox_row_number_fuzzer
	retention-days: ${{ env.RETENTION }}

	- name: Upload topn row number fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: topn_row_number
	path: velox/_build/debug//velox/exec/fuzzer/velox_topn_row_number_fuzzer
	retention-days: ${{ env.RETENTION }}

	- name: Upload writer fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: writer
	path: velox/_build/debug/velox/functions/prestosql/fuzzer/velox_writer_fuzzer_test
	retention-days: ${{ env.RETENTION }}

	- name: Upload spatial join fuzzer
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: spatial_join_fuzzer
	path: velox/_build/debug/velox/exec/fuzzer/velox_spatial_join_fuzzer
	retention-days: ${{ env.RETENTION }}

	- name: Upload shared library
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: libvelox
	path: velox/_build/debug/lib/libvelox.so
	retention-days: ${{ env.RETENTION }}

	presto-fuzzer-run:
	name: Presto Fuzzer
	if: ${{ needs.compile.outputs.presto_bias != 'true' }}
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 30
	steps:
	- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
	if: github.event_name == 'pull_request'
	id: changes
	with:
	filters: \|
	presto:
	- 'velox/expression/!(test)**'
	- 'velox/exec/!(test)**'
	- 'velox/common/!(test)**'
	- 'velox/core/!(test)**'
	- 'velox/vector/!(test)**'

	- name: Set presto specific fuzzer duration
	env:
	# Run for 10 minutes per instance instead of 5, when files relevant to presto are touched
	pr_duration: ${{ steps.changes.outputs.presto == 'true' && 600 \|\| 300 }}
	# Run for 20 minutes per instance if its a scheduled run
	other_duration: ${{ inputs.duration \|\| (github.event_name == 'push' && 600 \|\| 1200) }}
	is_pr: ${{ github.event_name == 'pull_request' }}
	run: \|
	if [ "$is_pr" == "true" ]; then
	duration=$pr_duration
	else
	duration=$other_duration
	fi

	echo "DURATION=$duration" >> $GITHUB_ENV

	- name: Download presto fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: presto

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox

	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run Presto Fuzzer
	run: \|
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_expression_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/fuzzer_repro \
	./velox_expression_fuzzer_test \
	--seed __SEED__ \
	--enable_variadic_signatures \
	--velox_fuzzer_enable_complex_types \
	--velox_fuzzer_enable_decimal_type \
	--lazy_vector_generation_ratio 0.2 \
	--common_dictionary_wraps_generation_ratio=0.3 \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--max_expression_trees_per_step 2 \
	--retry_with_try \
	--special_forms="and,or,cast,coalesce" \
	--enable_dereference \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--repro_persist_path=__REPRO_DIR__

	- name: Archive production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-fuzzer-failure-artifacts
	path: \|
	/tmp/fuzzer_repro

	presto-sot-fuzzer-run:
	name: Expression Fuzzer with Presto SOT
	needs: compile
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 30
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos
	steps:

	- name: Download Presto expression fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: presto

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox

	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run Presto expression fuzzer with Presto as a source of truth
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_expression_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/fuzzer_repro \
	./velox_expression_fuzzer_test \
	--seed __SEED__ \
	--enable_variadic_signatures \
	--velox_fuzzer_enable_complex_types \
	--lazy_vector_generation_ratio 0.2 \
	--common_dictionary_wraps_generation_ratio=0.3 \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--enable_dereference \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--repro_persist_path=__REPRO_DIR__ \
	--special_forms="cast,coalesce,if" \
	--velox_fuzzer_max_level_of_nesting=1 \
	--presto_url=http://127.0.0.1:8080 \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive expression production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-sot-fuzzer-failure-artifacts
	path: \|
	/tmp/fuzzer_repro

	presto-bias-fuzzer:
	name: Presto Bias Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	if: ${{ needs.compile.outputs.presto_bias == 'true' }}
	timeout-minutes: 120
	steps:

	- name: Download presto expression fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: presto

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Download Signatures
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: signatures
	path: /tmp/signatures

	- name: Run Presto Expression Fuzzer
	run: \|
	ls /tmp/signatures
	mkdir -p /tmp/presto_bias_fuzzer_repro/logs/
	chmod -R 777 /tmp/presto_bias_fuzzer_repro
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_expression_fuzzer_test
	random_seed=${RANDOM}
	echo "Random seed: ${random_seed}"
	echo "Biased functions: $(< /tmp/signatures/presto_bias_functions)"
	./velox_expression_fuzzer_test \
	--seed ${random_seed} \
	--lazy_vector_generation_ratio 0.2 \
	--common_dictionary_wraps_generation_ratio=0.3 \
	--assign_function_tickets $(cat /tmp/signatures/presto_bias_functions) \
	--duration_sec 3600 \
	--enable_variadic_signatures \
	--velox_fuzzer_enable_complex_types \
	--velox_fuzzer_enable_decimal_type \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--max_expression_trees_per_step 2 \
	--retry_with_try \
	--enable_dereference \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/presto_bias_fuzzer_repro/logs \
	--repro_persist_path=/tmp/presto_bias_fuzzer_repro \
	&& echo -e "\n\nPresto Fuzzer run finished successfully."

	- name: Archive Presto Bias expression production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-bias-fuzzer-failure-artifacts
	path: \|
	/tmp/presto_bias_fuzzer_repro

	spark-aggregate-fuzzer-run:
	name: Spark Aggregate Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:spark-server
	needs: compile
	timeout-minutes: 30
	steps:

	- name: Download spark aggregation fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: spark_aggregation_fuzzer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run Spark Aggregate Fuzzer
	run: \|
	bash /opt/start-spark.sh
	# Sleep for 120 seconds to allow Spark server to start.
	sleep 120
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x spark_aggregation_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/spark_aggregate_fuzzer_repro \
	./spark_aggregation_fuzzer_test \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--enable_sorted_aggregations=false \
	--enable_streaming_aggregations=false \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--repro_persist_path=__REPRO_DIR__ \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive Spark aggregate production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: spark-agg-fuzzer-failure-artifacts
	path: \|
	/tmp/spark_aggregate_fuzzer_repro

	spark-bias-fuzzer:
	name: Spark Bias Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	if: ${{ needs.compile.outputs.spark_bias == 'true' }}
	timeout-minutes: 120
	steps:

	- name: Download spark expression fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: spark_expression_fuzzer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Download Signatures
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: signatures
	path: /tmp/signatures

	- name: Run Spark Expression Fuzzer
	run: \|
	ls /tmp/signatures
	mkdir -p /tmp/spark_bias_fuzzer_repro/logs/
	chmod -R 777 /tmp/spark_bias_fuzzer_repro
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x spark_expression_fuzzer_test
	random_seed=${RANDOM}
	echo "Random seed: ${random_seed}"
	echo "Biased functions: $(< /tmp/signatures/spark_bias_functions)"
	./spark_expression_fuzzer_test \
	--seed ${random_seed} \
	--duration_sec $DURATION \
	--velox_fuzzer_enable_decimal_type \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/spark_bias_fuzzer_repro/logs \
	--assign_function_tickets $(cat /tmp/signatures/spark_bias_functions) \
	--repro_persist_path=/tmp/spark_bias_fuzzer_repro \
	&& echo -e "\n\nSpark Fuzzer run finished successfully."

	- name: Archive Spark expression production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: spark-fuzzer-failure-artifacts
	path: \|
	/tmp/spark_bias_fuzzer_repro

	spark-fuzzer:
	name: Spark Fuzzer
	if: ${{ needs.compile.outputs.spark_bias != 'true' }}
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 30
	steps:

	- name: Download spark expression fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: spark_expression_fuzzer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run Spark Expression Fuzzer
	run: \|
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x spark_expression_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/spark_fuzzer_repro \
	./spark_expression_fuzzer_test \
	--seed __SEED__ \
	--enable_variadic_signatures \
	--lazy_vector_generation_ratio 0.2 \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--max_expression_trees_per_step 2 \
	--retry_with_try \
	--enable_dereference \
	--velox_fuzzer_enable_decimal_type \
	--special_forms="and,or,cast,coalesce" \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--repro_persist_path=__REPRO_DIR__

	- name: Archive Spark expression production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: spark-fuzzer-failure-artifacts
	path: \|
	/tmp/spark_fuzzer_repro

	presto-java-join-fuzzer-run:
	name: Join Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	needs: compile
	timeout-minutes: 30
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos
	steps:

	- name: Download join fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: join

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run Join Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_join_fuzzer run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/join_fuzzer_repro \
	./velox_join_fuzzer \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--presto_url=http://127.0.0.1:8080 \
	--req_timeout_ms=2000 \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive join production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-sot-join-fuzzer-failure-artifacts
	path: \|
	/tmp/join_fuzzer_repro
	/tmp/server.log

	exchange-fuzzer-run:
	name: Exchange Fuzzer
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 30
	steps:

	- name: Download exchange fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: exchange

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run exchange Fuzzer
	run: \|
	cat /proc/sys/vm/max_map_count
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_exchange_fuzzer run-fuzzer-parallel.sh
	# Exchange fuzzer is multi-threaded and memory-heavy; run 1 instance
	# on ubuntu-latest with an extended duration.
	./run-fuzzer-parallel.sh 1 /tmp/exchange_fuzzer_repro \
	./velox_exchange_fuzzer \
	--seed __SEED__ \
	--duration_sec 480 \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--repro_path=__REPRO_DIR__

	- name: Archive Exchange production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: exchange-fuzzer-failure-artifacts
	path: \|
	/tmp/exchange_fuzzer_repro

	presto-java-row-number-fuzzer-run:
	name: RowNumber Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	needs: compile
	timeout-minutes: 30
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos

	steps:

	- name: Download row number fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: row_number

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run RowNumber Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_row_number_fuzzer run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/row_fuzzer_repro \
	./velox_row_number_fuzzer \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--presto_url=http://127.0.0.1:8080 \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive row number production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: row-fuzzer-failure-artifacts
	path: \|
	/tmp/row_fuzzer_repro

	presto-java-topn-row-number-fuzzer-run:
	name: TopNRowNumber Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	needs: compile
	timeout-minutes: 30
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos

	steps:

	- name: Download topn row number fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: topn_row_number

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run TopNRowNumber Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_topn_row_number_fuzzer run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/topn_row_fuzzer_repro \
	./velox_topn_row_number_fuzzer \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--req_timeout_ms 50000 \
	--log_dir=__LOG_DIR__ \
	--presto_url=http://127.0.0.1:8080 \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive topn row number production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: topn-row-fuzzer-failure-artifacts
	path: \|
	/tmp/topn_row_fuzzer_repro

	cache-fuzzer-run:
	name: Cache Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 30
	# Temporarily disable on PRs till flakiness is fixed #12167
	if: ${{ github.event_name != 'pull_request' }}
	steps:

	- name: Download cache fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: cache_fuzzer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run Cache Fuzzer
	run: \|
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_cache_fuzzer run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/cache_fuzzer \
	./velox_cache_fuzzer \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__

	- name: Archive Cache production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: cache-fuzzer-logs
	path: \|
	/tmp/cache_fuzzer

	table-evolution-fuzzer-run:
	name: Table Evolution Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 30
	steps:

	- name: Download table evolution fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: table_evolution_fuzzer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run Table Evolution Fuzzer
	run: \|
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_table_evolution_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/table_evolution_fuzzer_test \
	./velox_table_evolution_fuzzer_test \
	--seed __SEED__ \
	--table_evolution_fuzzer_duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__

	- name: Archive table evolution production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: table-evolution-fuzzer-test-logs
	path: \|
	/tmp/table_evolution_fuzzer_test

	memory-arbitration-fuzzer-run:
	name: Memory Arbitration Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 30
	steps:

	- name: Download memory arbitration fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: memory_arbitration_fuzzer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run Memory Arbitration Fuzzer
	run: \|
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_memory_arbitration_fuzzer run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/memory_arbitration_fuzzer \
	./velox_memory_arbitration_fuzzer \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__

	- name: Archive memory arbitration production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: memory-arbitration-fuzzer-test-logs
	path: \|
	/tmp/memory_arbitration_fuzzer_test

	presto-java-aggregation-fuzzer-run:
	name: Aggregation Fuzzer with Presto as source of truth
	needs: compile
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 30
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos
	steps:

	- name: Download aggregation fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: aggregation

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run Aggregate Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_aggregation_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/aggregate_fuzzer_repro \
	./velox_aggregation_fuzzer_test \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--repro_persist_path=__REPRO_DIR__ \
	--enable_sorted_aggregations=true \
	--presto_url=http://127.0.0.1:8080 \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive aggregate production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-sot-aggregate-fuzzer-failure-artifacts
	path: \|
	/tmp/aggregate_fuzzer_repro
	/tmp/server.log

	presto-java-only-bias-function-expression-fuzzer-run:
	name: Biased Expression Fuzzer with Only Added/Updated Functions and Presto as source of truth
	needs: compile
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 120
	if: ${{ needs.compile.outputs.presto_bias == 'true' }}
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos
	steps:

	- name: Download presto expression fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: presto

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Download Signatures
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: signatures
	path: /tmp/signatures

	- name: Run Bias Expression Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	mkdir -p /tmp/presto_only_bias_function_fuzzer_repro/logs/
	chmod -R 777 /tmp/presto_only_bias_function_fuzzer_repro
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_expression_fuzzer_test
	echo "Biased functions: $(< /tmp/signatures/presto_bias_functions)"
	# Convert the list of function names with tickets into a list of function names only.
	function_names=""
	IFS=',' read -r -a array <<< $(cat /tmp/signatures/presto_bias_functions)
	for x in ${array[@]}; do
	if [ -n "$function_names" ]; then
	function_names+=","
	fi
	function_names+=$(echo $x \| cut -d '=' -f 1)
	done
	echo "Biased function names: $function_names"
	random_seed=${RANDOM}
	echo "Random seed: ${random_seed}"
	echo "Running Fuzzer for $DURATION"
	./velox_expression_fuzzer_test \
	--seed ${random_seed} \
	--lazy_vector_generation_ratio 0.2 \
	--common_dictionary_wraps_generation_ratio=0.3 \
	--only=$function_names \
	--duration_sec $DURATION \
	--enable_variadic_signatures \
	--velox_fuzzer_enable_complex_types \
	--velox_fuzzer_enable_column_reuse \
	--velox_fuzzer_enable_expression_reuse \
	--max_expression_trees_per_step 2 \
	--retry_with_try \
	--batch_size=6 \
	--presto_url=http://127.0.0.1:8080 \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/presto_only_bias_function_fuzzer_repro/logs \
	--repro_persist_path=/tmp/presto_only_bias_function_fuzzer_repro \
	&& echo -e "\n\nPresto Fuzzer run finished successfully."
	- name: Archive Presto only-bias-function expression fuzzer production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-only-bias-function-fuzzer-failure-artifacts
	path: \|
	/tmp/presto_only_bias_function_fuzzer_repro

	presto-bias-java-aggregation-fuzzer-run:
	name: Biased Aggregation Fuzzer with Presto as source of truth
	needs: compile
	runs-on: ubuntu-latest
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 120
	if: ${{ needs.compile.outputs.presto_aggregate_bias == 'true' }}
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos
	steps:

	- name: Download aggregation fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: aggregation

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Download Signatures
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: signatures
	path: /tmp/signatures

	- name: Run Bias Aggregate Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	mkdir -p /tmp/aggregate_fuzzer_repro/logs/
	chmod -R 777 /tmp/aggregate_fuzzer_repro
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_aggregation_fuzzer_test
	echo "signatures folder"
	ls /tmp/signatures/
	echo "Biased functions: $(< /tmp/signatures/presto_aggregate_bias_functions)"
	random_seed=${RANDOM}
	echo "Random seed: ${random_seed}"
	echo "Running Fuzzer for $DURATION"
	./velox_aggregation_fuzzer_test \
	--seed ${random_seed} \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=/tmp/aggregate_fuzzer_repro/logs \
	--repro_persist_path=/tmp/aggregate_fuzzer_repro \
	--enable_sorted_aggregations=true \
	--only=$(< /tmp/signatures/presto_aggregate_bias_functions) \
	--presto_url=http://127.0.0.1:8080 \
	&& echo -e "\n\nAggregation fuzzer run finished successfully."

	- name: Archive bias aggregate production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-bias-sot-aggregate-fuzzer-failure-artifacts
	path: \|
	/tmp/aggregate_fuzzer_repro
	/tmp/server.log

	surface-signature-errors:
	name: Signature Changes
	if: ${{ github.event_name != 'schedule' && needs.compile.outputs.signatures_checked != 'false' }}
	needs: compile
	runs-on: ubuntu-latest
	steps:
	- name: Download Signatures
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: signatures
	path: /tmp/signatures

	- name: Surface Presto function signature errors
	if: ${{ needs.compile.outputs.presto_error == 'true' }}
	run: \|
	cat /tmp/signatures/presto_errors
	exit 1

	- name: Surface Aggregate function signature errors
	if: ${{ needs.compile.outputs.presto_aggregate_error == 'true' }}
	run: \|
	cat /tmp/signatures/presto_aggregate_errors
	exit 1

	presto-java-window-fuzzer-run:
	name: Window Fuzzer with Presto as source of truth
	needs: compile
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 30
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos
	steps:

	- name: Download window fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: window

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run Window Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_window_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/window_fuzzer_repro \
	./velox_window_fuzzer_test \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--batch_size=50 \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__ \
	--repro_persist_path=__REPRO_DIR__ \
	--enable_window_reference_verification \
	--presto_url=http://127.0.0.1:8080 \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive window production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-sot-window-fuzzer-failure-artifacts
	path: \|
	/tmp/window_fuzzer_repro
	/tmp/server.log

	presto-java-writer-fuzzer-run:
	name: Writer Fuzzer with Presto as source of truth
	needs: compile
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:presto-java
	timeout-minutes: 30
	env:
	CCACHE_DIR: ${{ github.workspace }}/ccache/
	LINUX_DISTRO: centos
	steps:

	- name: Download writer fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: writer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Checkout Repo
	uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
	with:
	path: velox
	ref: ${{ inputs.ref }}
	persist-credentials: false

	- name: Fix git permissions
	# Usually actions/checkout does this but as we run in a container
	# it doesn't work
	run: git config --global --add safe.directory ${GITHUB_WORKSPACE}/velox

	- name: Run Writer Fuzzer
	run: \|
	cd velox
	cp ./scripts/ci/presto/etc/hive.properties $PRESTO_HOME/etc/catalog
	ls -lR $PRESTO_HOME/etc
	echo "jvm config content:"
	cat $PRESTO_HOME/etc/jvm.config
	$PRESTO_HOME/bin/launcher run -v > /tmp/server.log 2>&1 &
	ls -lR /var/log
	# Sleep for 60 seconds to allow Presto server to start.
	sleep 60
	/opt/presto-cli --version
	/opt/presto-cli --server 127.0.0.1:8080 --execute 'CREATE SCHEMA IF NOT EXISTS hive.tpch;'
	cd -
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_writer_fuzzer_test run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/writer_fuzzer_repro \
	./velox_writer_fuzzer_test \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--req_timeout_ms 60000 \
	--log_dir=__LOG_DIR__ \
	--presto_url=http://127.0.0.1:8080 \
	--table_name_prefix=i__INSTANCE_ID__

	- name: Archive writer production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: presto-sot-writer-fuzzer-failure-artifacts
	path: \|
	/tmp/writer_fuzzer_repro
	/tmp/server.log
	/var/log

	spatial-join-fuzzer-run:
	name: Spatial Join Fuzzer
	runs-on: 4-core-ubuntu
	container: ghcr.io/facebookincubator/velox-dev:centos9
	needs: compile
	timeout-minutes: 30
	steps:

	- name: Download spatial join fuzzer
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: spatial_join_fuzzer

	- name: Download libvelox
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: libvelox
	- name: Download fuzzer scripts
	uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
	with:
	name: fuzzer-scripts

	- name: Run Spatial Join Fuzzer
	run: \|
	export LD_LIBRARY_PATH=$(pwd):$LD_LIBRARY_PATH
	chmod +x velox_spatial_join_fuzzer run-fuzzer-parallel.sh
	./run-fuzzer-parallel.sh $NUM_FUZZER_INSTANCES /tmp/spatial_join_fuzzer_repro \
	./velox_spatial_join_fuzzer \
	--seed __SEED__ \
	--duration_sec $DURATION \
	--minloglevel=0 \
	--stderrthreshold=2 \
	--log_dir=__LOG_DIR__

	- name: Archive spatial join fuzzer production artifacts
	if: ${{ !cancelled() }}
	uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
	with:
	name: spatial-join-fuzzer-failure-artifacts
	path: \|
	/tmp/spatial_join_fuzzer_repro

	linux-clang:
	if: ${{ (github.event_name == 'schedule') \|\| (github.event_name == 'workflow_dispatch') }}
	name: Build with Clang
	uses: ./.github/workflows/linux-build-base.yml
	with:
	use-clang: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: Add executor pool size to S3 configs #41276

Workflow file

feat: Add executor pool size to S3 configs #41276

Uh oh!

Workflow file for this run