Use encoder-only method for reduced-tx-part-set option #858

Workflow file for this run

.github/workflows/pull_request.yaml at bd27a66

	name: Pull Request
	on:
	pull_request:
	workflow_dispatch:

	defaults:
	run:
	shell: bash

	concurrency:
	group: pr-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	# This job is a hack because GitHub doesn't support referencing env vars for setting runners or container image
	# See https://github.com/orgs/community/discussions/26324
	vars:
	name: Set common variables
	runs-on: ubuntu-latest
	outputs:
	runner: self-hosted
	container-image: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404:20260116205104
	container-image-multilib: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404-multilib:20260116205104
	timeout-minutes: 660
	steps:
	- name: Do nothing
	run: \|
	echo noop

	style-checks:
	name: Style Checks
	if: (!cancelled())
	needs:
	- vars
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	DIFF_REF: ${{ github.event.pull_request.base.sha }}
	steps:
	- uses: actions/checkout@v5
	with:
	fetch-depth: 50
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Run clang-format
	run: \|
	echo "lang-format version: $(clang-format --version)"

	# Run clang-format check.
	for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '.[hc]pp' '.cc' '*.[ch]' \| grep -v third_party); do
	exit_code=0
	clang-format -i --style=file $f -n -Werror \|\| exit_code=$?
	if [ ${exit_code} -ne 0 ]; then
	echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
	echo -e "\e[31mhttps://github.com/AOMediaCodec/avm/wiki/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
	exit 1
	fi
	done

	- name: Run cmake-format
	run: \|
	echo "cmake-format version: $(cmake-format --version)"

	# Run cmake-format check.
	for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '*.cmake' 'CMakeLists.txt' \| grep -v third_party); do
	exit_code=0
	cmake-format --check $f \|\| exit_code=$?
	if [ ${exit_code} -ne 0 ]; then
	echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
	echo -e "\e[31mhttps://github.com/AOMediaCodec/avm/wiki/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
	exit 1
	fi
	done

	pr-checks:
	name: PR Checks
	if: (!cancelled())
	needs:
	- vars
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	DIFF_REF: ${{ github.event.pull_request.base.sha }}
	steps:
	- uses: actions/checkout@v5
	with:
	fetch-depth: 50
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Validate files with the executable bit set
	run: \|
	echo lol=${{ github.ref }}
	git diff "${DIFF_REF}" --check

	# Validate files with the executable bit set
	files=$(git diff --name-only "${DIFF_REF}" \| tr '\n' ' ')
	git ls-tree -r HEAD $files \| while read mode type sha1 file; do
	if [ "$mode" = "100755" ]; then
	case "$file" in
	configure\|.php\|.pl\|.py\|.sh)
	;;
	*)
	echo "File $file should not be executable."
	echo "Only configure\|.php\|.pl\|.py\|.sh are accepted."
	exit 1
	;;
	esac
	fi
	done

	test-data-checks:
	name: Test Data Checks
	if: (!cancelled())
	needs:
	- vars
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	DIFF_REF: ${{ github.event.pull_request.base.sha }}
	steps:
	- uses: actions/checkout@v5
	with:
	fetch-depth: 50
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Ensure added test files add declared in test data
	run: \|
	# TODO: replace this test with a proper job/rule filter
	if [ -z "$(git diff ${DIFF_REF} "test/")" ]; then
	echo "No test file added."
	exit 0
	fi
	ADDED_TEST_FILES="$(git diff ${DIFF_REF} "test/" \
	\| perl -ne '/^\+.?"((?!\$).?\.(ivf\|webm\|res\|mkv\|y4m\|yuv))"/g and print "$1\n"')" \
	printf "Checking for files:\n%s\n\n" "$ADDED_TEST_FILES"
	echo "$ADDED_TEST_FILES" \| while read -r f; do
	if [ -n "$(grep -L "${f}" test/test-data.sha1 test/test_data_util.cmake)" ]; then
	echo "file: ${f} was not found in test-data.sha1 or test_data_util.cmake"
	exit 1
	fi
	done

	previous-build-x86_64-linux-gcc:
	name: Previous Build (x86_64-linux-gcc)
	uses: ./.github/workflows/build-job-reusable.yaml
	needs:
	- style-checks
	- pr-checks
	- test-data-checks
	- vars
	if: (!cancelled())
	with:
	avm-build-config: ${{ matrix.avm-build-config }}
	parent-job-name: previous-build-x86_64-linux-gcc
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}
	show-github-context: true
	extra-env-vars: \|
	DIFF_REF=${{ github.event.pull_request.base.sha }}
	INSTALLROOT_FOLDER=installroot_old
	# Allow build warnings.
	EXTRA_CMAKE_FLAGS=-DENABLE_WERROR=0
	before-script: \|
	#TODO: git fetch --unshallow
	git checkout "${DIFF_REF}"
	artifact-paths: /installroot_old
	strategy:
	matrix:
	avm-build-config:
	- encode-only

	test-data:
	name: Test Data
	uses: ./.github/workflows/common-test-data-reusable.yaml
	needs:
	- vars
	if: (!cancelled())
	with:
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}

	common-builds:
	name: Common Builds
	uses: ./.github/workflows/common-builds-reusable.yaml
	needs:
	- style-checks
	- pr-checks
	- test-data-checks
	- vars
	if: (!cancelled())
	with:
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}
	container-image-multilib: ${{ needs.vars.outputs.container-image-multilib }}

	example-test-x86_64-gcc:
	name: Example Test (x86_64-gcc)
	needs:
	- common-builds
	- test-data
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	LIBAVM_TEST_DATA_PATH: ${{ github.workspace }}/libavm-test-data
	steps:
	- uses: actions/checkout@v5
	with:
	fetch-depth: 0
	fetch-tags: true
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Get test data
	uses: actions/download-artifact@v8
	with:
	name: test-data
	path: libavm-test-data

	- name: Get example build
	uses: actions/download-artifact@v8
	with:
	name: example-build
	path: avm_example_build

	- name: Fix executable bit permissions
	run: \|
	# See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

	# Look for files starting with ELF header and set executable but
	for file in $(find avm_example_build/ -type f); do
	if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
	echo "Found binary ${file}, fixing permissions."
	chmod a+x "${file}"
	fi
	done

	- name: Run example test
	run: \|
	cpu_cores=$(nproc)
	commands=/tmp/parallel.commands.$$
	ret=0

	printf "" >${commands}

	cd avm_example_build
	for test in $(../test/examples.sh --list-tests); do
	echo "${GITHUB_WORKSPACE}/.github/capture-logs.sh ${GITHUB_WORKSPACE}/example.test.${test}.log sh ../test/examples.sh --bin-path examples --verbose --show-program-output --filter \"\b${test}\b\"" >>${commands}
	done

	cat ${commands}
	time parallel --line-buffer --jobs ${cpu_cores} < ${commands} \|\| ret=$?

	if [ ${ret} -ne 0 ]; then
	echo "Failure running example tests, see logs for details." >&2
	exit 1
	fi

	- name: Upload logs
	uses: actions/upload-artifact@v7
	if: ${{ always() }}
	with:
	name: example-test-logs
	path: example.test.*.log

	linux-sanitizer-test:
	name: Linux Sanitizer Test
	uses: ./.github/workflows/sanitizer-job-reusable.yaml
	needs:
	- common-builds
	- vars
	if: (!cancelled())
	with:
	avm-sanitizer-type: ${{ matrix.avm-sanitizer-type }}
	max-total-shards: ${{ matrix.max-total-shards }}
	runner: '["${{ needs.vars.outputs.runner }}"]'
	container-image: ${{ needs.vars.outputs.container-image }}
	show-github-context: true
	strategy:
	fail-fast: false
	matrix:
	avm-sanitizer-type:
	- address
	- integer
	- thread
	- undefined
	# CFI Sanitizer commented for now, as lto build does not work
	# - cfi
	include:
	# default to unlimited
	- max-total-shards: 0

	encdec-run:
	name: Enc/Dec
	needs:
	- common-builds
	- previous-build-x86_64-linux-gcc
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	AVMENC_OUTPUT: ${{ matrix.avm-enc-output }}
	AVMENC_LIMIT: ${{ matrix.avm-enc-limit }}
	AVMENC_QP: 210
	AVMENC_INPUT: Vertical_Bayshore_270x480_2997.y4m
	AVMENC_COMMON_ARGS: "\
	--debug \
	--cpu-used=0 \
	--passes=1 \
	--deltaq-mode=0 \
	--enable-keyframe-filtering=0 \
	--enable-tpl-model=0 \
	--use-fixed-qp-offsets=1 \
	--end-usage=q \
	--obu \
	--psnr \
	"

	steps:
	- uses: actions/checkout@v5
	with:
	fetch-depth: 50

	- uses: ./.github/actions/common-setup

	- name: Configure AVM encoder
	run: \|
	if [ -z "${AVMENC_OUTPUT}" ]; then
	echo "AVMENC_OUTPUT is not set." >&2
	exit 1
	fi

	# common encoder arguments to all outputs
	AVMENC_ARGS="${AVMENC_COMMON_ARGS} \
	--qp=${AVMENC_QP} \
	--limit=${AVMENC_LIMIT} \
	"

	# output-specific arguments
	case ${AVMENC_OUTPUT} in
	all-intra)
	AVMENC_ARGS="${AVMENC_ARGS} \
	--kf-max-dist=0 \
	--kf-min-dist=0 \
	"
	;;

	random-access)
	AVMENC_ARGS="${AVMENC_ARGS} \
	--auto-alt-ref=1 \
	--gf-max-pyr-height=4 \
	--gf-min-pyr-height=4 \
	--kf-max-dist=65 \
	--kf-min-dist=65 \
	--lag-in-frames=19 \
	--max-gf-interval=16 \
	--min-gf-interval=16 \
	"
	;;

	low-delay)
	AVMENC_ARGS="${AVMENC_ARGS} \
	--gf-max-pyr-height=4 \
	--gf-min-pyr-height=4 \
	--kf-max-dist=9999 \
	--kf-min-dist=9999 \
	--lag-in-frames=0 \
	--subgop-config-str=ld \
	"
	;;

	*)
	echo "Unknown encoder output: '${AVMENC_OUTPUT}'" >&2
	exit 1
	;;
	esac

	# Append to workflow environment
	echo "AVMENC_ARGS=${AVMENC_ARGS}" >>${GITHUB_ENV}

	- name: Pull input file
	run: \|
	curl -s -S -f -O https://storage.googleapis.com/aom-test-data/${AVMENC_INPUT}.xz
	unxz ${AVMENC_INPUT}.xz

	- name: Pull previous encoder build
	uses: actions/download-artifact@v8
	with:
	name: previous-build-x86_64-linux-gcc-encode-only
	path: enc-previous

	- name: Pull current encoder build
	uses: actions/download-artifact@v8
	with:
	name: example-build
	path: enc-current

	- name: Fix executable bit permissions
	run: \|
	# See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

	# Look for files starting with ELF header and set executable but
	for file in $(find . -type f); do
	if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
	echo "Found binary ${file}, fixing permissions."
	chmod a+x "${file}"
	fi
	done

	- name: Encode
	run: \|
	printf -- "\n---------------------------------------------------\n"
	echo "Encoder input: ${AVMENC_INPUT}"
	echo "Encoder output: ${AVMENC_OUTPUT}"
	echo "Encoder args: ${AVMENC_ARGS}"
	printf -- "---------------------------------------------------\n\n"

	commands=/tmp/parallel.commands.$$
	cat >${commands} <<EOF
	enc-previous/usr/local/bin/avmenc ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.previous.obu ${AVMENC_INPUT} 2>&1 \| tee ${AVMENC_OUTPUT}.previous.psnr.log
	enc-current/avmenc ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.current.obu ${AVMENC_INPUT} 2>&1 \| tee ${AVMENC_OUTPUT}.current.psnr.log
	EOF

	time parallel --line-buffer < ${commands} \|\| true

	for obu_file in ${AVMENC_OUTPUT}.previous.obu ${AVMENC_OUTPUT}.current.obu; do
	if [ ! -f ${obu_file} ]; then
	echo "Cannot find OBU file ${obu_file}" >&2
	exit 1
	fi

	md5sum --binary ${obu_file} \| awk '{ print $1; }' > ${obu_file}.md5
	done

	- name: Decode
	run: \|
	enc-current/avmdec \
	${AVMENC_OUTPUT}.current.obu \
	--output=${AVMDEC_OUTPUT}.current.decoded.y4m \
	--summary \
	2>&1 \| tee ${AVMDEC_OUTPUT}.current.summary.log

	if [ ! -f ${AVMDEC_OUTPUT}.current.decoded.y4m ]; then
	echo "Cannot find decoded Y4M file" >&2
	exit 1
	fi

	if [ ! -f ${AVMDEC_OUTPUT}.current.summary.log ]; then
	echo "Cannot find decoder summary log file" >&2
	exit 1
	fi

	for str in 'decoded frames' 'showed frames'; do
	frame_count=$(grep -E -o "[0-9]+ ${str}" "${AVMDEC_OUTPUT}.current.summary.log" \| sed -E "s/([0-9]+) ${str}/\1/g")
	echo "${str} = ${frame_count}"
	if [ ${frame_count} -ne ${AVMENC_LIMIT} ]; then
	echo "ERROR: Unexpected number of ${str}. Got ${frame_count}, expected ${AVMENC_LIMIT}" >&2
	exit 1
	fi
	done

	- name: Upload artifacts
	uses: actions/upload-artifact@v7
	if: ${{ always() }}
	with:
	name: encdec-${{ matrix.avm-enc-output }}
	path: ${{ matrix.avm-enc-output }}.*

	- name: Compare hashes
	run: \|
	ALLOW_FAIL=0
	FAILED=0

	if [ -n "$(git --no-pager log --grep STATS_CHANGED --format=format:"%H" "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}")" ]; then
	ALLOW_FAIL=1
	fi

	variant=${AVMDEC_OUTPUT}

	if diff -- "${AVMENC_OUTPUT}.previous.obu.md5" "${AVMENC_OUTPUT}.current.obu.md5" &>/dev/null; then
	echo "OK: No differences in ${AVMENC_OUTPUT} outputs found."
	echo ""
	else
	echo "WARNING: ${AVMENC_OUTPUT}.previous.obu.md5 and ${AVMENC_OUTPUT}.current.obu.md5 differ!"
	echo "${AVMENC_OUTPUT}.previous.obu.md5 : $(cat ${AVMENC_OUTPUT}.previous.obu.md5)"
	echo "${AVMENC_OUTPUT}.current.obu.md5 : $(cat ${AVMENC_OUTPUT}.current.obu.md5)"
	echo ""
	FAILED=1
	fi

	if [ ${FAILED} -eq 1 ]; then
	if [ ${ALLOW_FAIL} -eq 1 ]; then
	echo "SUCCESS: Even though a mismatch was detected, one or more"
	echo " commits contain the STATS_CHANGED keyword, so mismatch"
	echo " of encode outputs is allowed."
	echo "Commits containing STATS_CHANGED added in this MR:"
	git --no-pager log --grep STATS_CHANGED --format=format:"%h: %s; \"%aN\" <%aE>" "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}"
	echo ""
	exit 0
	fi

	echo "FAIL: Failing job due to mismatch and no STATS_CHANGED keyword"
	echo " was found in any of the commits added in this MR!"
	exit 1
	fi

	strategy:
	fail-fast: false
	matrix:
	avm-enc-output:
	- all-intra
	- random-access
	- low-delay
	include:
	- avm-enc-limit: 30
	- avm-enc-limit: 15
	avm-enc-output: all-intra

	static-analyzer:
	name: Static Analyzer
	needs:
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	env:
	ANALYZER_MODE: ${{ matrix.analyzer-mode }}
	steps:
	- uses: actions/checkout@v5
	with:
	fetch-depth: 50
	fetch-tags: true
	lfs: true

	- uses: ./.github/actions/common-setup

	- name: Analyze
	run: \|
	mkdir analyzer-${ANALYZER_MODE}

	scan_build() {
	scan-build \
	--exclude third_party \
	--exclude _deps \
	--exclude abseil-cpp \
	--exclude benchmark \
	--exclude cpuinfo \
	--exclude eigen \
	--exclude farmhash \
	--exclude fft2d \
	--exclude flatbuffers \
	--exclude flatbuffers-flatc \
	--exclude fp16 \
	--exclude FP16 \
	--exclude FXdiv \
	--exclude psimd \
	--exclude ml_dtypes \
	--exclude neon2sse \
	--exclude protobuf \
	--exclude pthreadpool \
	--exclude pthreadpool-source \
	--exclude ruy \
	--exclude xnnpack \
	--exclude XNNPACK \
	--exclude googletest \
	-o analyzer-${ANALYZER_MODE} \
	-analyzer-config mode=${ANALYZER_MODE} \
	$*
	}

	scan_build cmake -B avm_build -GNinja -DCMAKE_BUILD_TYPE=Debug
	exit_code=0
	scan_build --status-bugs cmake --build avm_build \|\| exit_code=$?
	if [ ${exit_code} -ne 0 ]; then
	echo -e "\e[31mMore info on static analysis can be found at the following this wiki page:\e[0m"
	echo -e "\e[31mhttps://github.com/AOMediaCodec/avm/wiki/Reproducing-CI-Test-Failures-Locally#static-analysis\e[0m"
	exit 1
	fi

	- name: Upload artifacts
	uses: actions/upload-artifact@v7
	if: ${{ failure() }}
	with:
	name: analyzer-${{ matrix.analyzer-mode }}
	path: analyzer-${{ matrix.analyzer-mode }}
	retention-days: 14

	strategy:
	fail-fast: false
	matrix:
	analyzer-mode:
	- deep
	- shallow

	so-checks:
	name: SO checks
	needs:
	- common-builds
	- vars
	if: (!cancelled())
	timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
	runs-on: ${{ needs.vars.outputs.runner }}
	container:
	image: ${{ needs.vars.outputs.container-image }}
	steps:
	- name: Pull build
	uses: actions/download-artifact@v8
	with:
	name: build-x86_64-linux-gcc-shared

	- name: Check SO
	run: \|
	for so in $(find -name '.so'); do
	echo "Checking ${so} for textrels..."
	if readelf -dW ${so} \| grep TEXTREL; then
	textrels=$(eu-findtextrel ${so} \| sort \| uniq)
	if [ -n "${textrels}" ]; then
	echo "${textrels}"
	exit 1
	fi
	fi

	echo "Checking ${so} ELF header/section headers..."
	if readelf -lW ${so} \| grep WE; then
	echo "Invalid ELF header/section headers"
	echo "https://android.googlesource.com/platform/bionic/+/master/android-changes-for-ndk-developers.md#Writable-and-Executable-Segments-Enforced-for-API-level-26"
	exit 1
	fi
	done

Navigation Menu

Uh oh!

Use encoder-only method for reduced-tx-part-set option #858

Workflow file

Use encoder-only method for reduced-tx-part-set option #858

Uh oh!

Workflow file for this run