[pull] main from Comfy-Org:main #59

Workflow file for this run

.github/workflows/pr-update-playwright-expectations.yaml at 296fc5c

	# Setting test expectation screenshots for Playwright
	name: 'PR: Update Playwright Expectations'

	on:
	pull_request:
	types: [labeled]
	issue_comment:
	types: [created]

	# github.ref is the default branch for issue_comment events. Both triggers use
	# the PR number because they push snapshots to the same branch.
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.issue.number \|\| github.event.number \|\| github.ref }}
	cancel-in-progress: true

	jobs:
	setup:
	runs-on: ubuntu-latest
	if: >
	( github.event_name == 'pull_request' && github.event.label.name == 'New Browser Test Expectations' ) \|\|
	( github.event.issue.pull_request &&
	github.event_name == 'issue_comment' &&
	(
	github.event.comment.author_association == 'OWNER' \|\|
	github.event.comment.author_association == 'MEMBER' \|\|
	github.event.comment.author_association == 'COLLABORATOR'
	) &&
	startsWith(github.event.comment.body, '/update-playwright') )
	outputs:
	pr-number: ${{ steps.pr-info.outputs.pr-number }}
	branch: ${{ steps.pr-info.outputs.branch }}
	comment-id: ${{ steps.find-update-comment.outputs.comment-id }}
	container-image: ${{ steps.container-image.outputs.image }}
	steps:
	- name: Get PR info
	id: pr-info
	run: \|
	echo "pr-number=${{ github.event.number \|\| github.event.issue.number }}" >> $GITHUB_OUTPUT
	echo "branch=$(gh pr view ${{ github.event.number \|\| github.event.issue.number }} --repo ${{ github.repository }} --json headRefName --jq '.headRefName')" >> $GITHUB_OUTPUT
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Find Update Comment
	uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0
	id: 'find-update-comment'
	with:
	issue-number: ${{ steps.pr-info.outputs.pr-number }}
	comment-author: 'github-actions[bot]'
	body-includes: 'Updating Playwright Expectations'

	- name: Add Starting Reaction
	uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
	with:
	comment-id: ${{ steps.find-update-comment.outputs.comment-id }}
	issue-number: ${{ steps.pr-info.outputs.pr-number }}
	body: \|
	Updating Playwright Expectations
	edit-mode: replace
	reactions: eyes

	- name: Checkout repository
	uses: actions/checkout@v7
	with:
	ref: ${{ steps.pr-info.outputs.branch }}

	# issue_comment loads this workflow from the default branch. Resolve the
	# image from the checked-out PR so regenerated snapshots match its CI.
	- name: Resolve CI container image from PR branch
	id: container-image
	shell: bash
	run: \|
	set -euo pipefail

	e2e_workflow='.github/workflows/ci-tests-e2e.yaml'
	if [ ! -f "$e2e_workflow" ]; then
	echo "::error::${e2e_workflow} not found on this PR branch; cannot resolve the CI container image"
	exit 1
	fi

	# Require every E2E container to use the same static image.
	mapfile -t images < <(
	grep -oE "^[[:space:]]image:[[:space:]]['\"]?ghcr\.io/comfy-org/comfyui-ci-container:[A-Za-z0-9._-]+" "$e2e_workflow" \|
	grep -oE 'ghcr\.io/comfy-org/comfyui-ci-container:[A-Za-z0-9._-]+' \|
	sort -u
	)
	if [ "${#images[@]}" -ne 1 ]; then
	echo "::error::Expected exactly one comfyui-ci-container image across the container jobs in ${e2e_workflow}, found ${#images[@]}: ${images[*]:-none}"
	exit 1
	fi

	echo "Regenerating snapshots with ${images[0]} (resolved from the PR branch)"
	echo "image=${images[0]}" >> "$GITHUB_OUTPUT"

	# Other workflow changes still come from the default branch on this path.
	- name: Warn when this workflow differs from the default branch
	if: github.event_name == 'issue_comment'
	shell: bash
	env:
	DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
	run: \|
	set -euo pipefail

	workflow='.github/workflows/pr-update-playwright-expectations.yaml'

	if ! git fetch --depth=1 origin "$DEFAULT_BRANCH"; then
	echo "::notice::Could not fetch ${DEFAULT_BRANCH}; skipping workflow drift check"
	exit 0
	fi

	if ! git diff --quiet FETCH_HEAD -- "$workflow"; then
	echo "::warning::This PR modifies ${workflow}, but /update-playwright runs the ${DEFAULT_BRANCH} copy of it. Only the container image is taken from the PR; other changes to this workflow are NOT in effect for this run. Use the 'New Browser Test Expectations' label instead to run the PR's own copy."
	fi

	- name: Setup frontend
	uses: ./.github/actions/setup-frontend
	with:
	include_build_step: true

	# Upload built dist/ (containerized test jobs will pnpm install without cache)
	- name: Upload built frontend
	uses: actions/upload-artifact@v6
	with:
	name: frontend-dist
	path: dist/
	retention-days: 1

	- name: Build cloud frontend
	uses: ./.github/actions/build-cloud-frontend

	# Shards 1-4 run against the plain build; the `cloud` leg runs @cloud
	# tests against the frontend-dist-cloud build.
	update-snapshots-sharded:
	needs: setup
	runs-on: ubuntu-latest
	container:
	image: ${{ needs.setup.outputs.container-image }}
	credentials:
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}
	permissions:
	contents: read
	packages: read
	strategy:
	fail-fast: false
	matrix:
	shard: [1, 2, 3, 4, cloud]
	steps:
	- name: Checkout repository
	uses: actions/checkout@v7
	with:
	ref: ${{ needs.setup.outputs.branch }}
	- name: Download built frontend
	uses: actions/download-artifact@v8
	with:
	name: ${{ matrix.shard == 'cloud' && 'frontend-dist-cloud' \|\| 'frontend-dist' }}
	path: dist/

	- name: Start ComfyUI server
	uses: ./.github/actions/start-comfyui-server

	- name: Install frontend deps
	run: pnpm install --frozen-lockfile

	# Snapshot rewrites fail the test command, so validate its JSON report.
	- name: Update snapshots (${{ matrix.shard == 'cloud' && 'cloud' \|\| format('Shard {0}/4', matrix.shard) }})
	id: playwright-tests
	env:
	PLAYWRIGHT_JSON_OUTPUT_NAME: /tmp/playwright-results.json
	run: pnpm exec playwright test --update-snapshots --reporter=json,html --grep @screenshot ${{ matrix.shard == 'cloud' && '--project=cloud' \|\| format('--grep-invert @cloud --shard={0}/4', matrix.shard) }}
	continue-on-error: true

	- name: Stage changed snapshot files
	id: changed-snapshots
	shell: bash
	run: \|
	set -euo pipefail

	# Configure git safe.directory for container environment
	git config --global --add safe.directory "$(pwd)"

	# Get list of changed snapshot files (including untracked/new files)
	changed_files=$( (
	git diff --name-only browser_tests/ 2>/dev/null \|\| true
	git ls-files --others --exclude-standard browser_tests/ 2>/dev/null \|\| true
	) \| sort -u \| grep -E '\-snapshots/' \|\| true )

	if [ -z "$changed_files" ]; then
	echo "No snapshot changes in shard ${{ matrix.shard }}"
	echo "has-changes=false" >> $GITHUB_OUTPUT
	exit 0
	fi

	file_count=$(echo "$changed_files" \| wc -l)
	echo "Shard ${{ matrix.shard }}: $file_count changed snapshot(s):"
	echo "$changed_files"
	echo "has-changes=true" >> $GITHUB_OUTPUT

	# Copy changed files to staging directory
	mkdir -p /tmp/changed_snapshots_shard
	while IFS= read -r file; do
	[ -f "$file" ] \|\| continue
	file_without_prefix="${file#browser_tests/}"
	mkdir -p "/tmp/changed_snapshots_shard/$(dirname "$file_without_prefix")"
	cp "$file" "/tmp/changed_snapshots_shard/$file_without_prefix"
	done <<< "$changed_files"

	# Reject test failures that produced no rewritten snapshots.
	- name: Verify shard did the work it claims
	shell: bash
	env:
	SHARD: ${{ matrix.shard }}
	HAS_CHANGES: ${{ steps.changed-snapshots.outputs.has-changes }}
	run: \|
	set -uo pipefail

	results='/tmp/playwright-results.json'

	# Playwright still writes a report when zero tests match.
	if [ ! -f "$results" ]; then
	echo "::error::Shard ${SHARD}: Playwright wrote no JSON report, so it never completed a run. Refusing to report success."
	exit 1
	fi

	# Node is available in the CI container; jq may not be.
	if ! summary=$(node -e '
	const fs = require("fs");
	const r = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
	const s = r.stats;
	if (!s) throw new Error("no stats in report");
	const n = (v) => (typeof v === "number" ? v : 0);
	const notPassing = n(s.unexpected) + n(s.flaky);
	const ran = n(s.expected) + notPassing;
	const errs = Array.isArray(r.errors) ? r.errors.length : 0;
	console.log(ran + " " + notPassing + " " + errs);
	console.log(JSON.stringify(s));
	' "$results" 2>&1); then
	echo "::error::Shard ${SHARD}: Playwright JSON report is unreadable or has no stats. Refusing to report success."
	echo "$summary"
	exit 1
	fi

	read -r ran not_passing error_count <<< "$(printf '%s\n' "$summary" \| head -1)"
	echo "Shard ${SHARD} stats: $(printf '%s\n' "$summary" \| tail -1)"

	# An empty grep result is valid despite Playwright's non-zero exit.
	if [ "$ran" -eq 0 ]; then
	echo "::notice::Shard ${SHARD}: no tests matched this shard's filters; nothing to regenerate."
	exit 0
	fi

	if [ "$error_count" -gt 0 ]; then
	echo "::error::Shard ${SHARD}: Playwright reported ${error_count} top-level error(s); the regenerated baselines may be incomplete."
	exit 1
	fi

	# Snapshot rewrites appear as unexpected or flaky tests.
	if [ "$not_passing" -gt 0 ] && [ "${HAS_CHANGES:-false}" != 'true' ]; then
	echo "::error::Shard ${SHARD}: ${not_passing} test(s) did not pass yet no snapshot was rewritten. That is a broken run, not an absence of drift. Refusing to report success."
	exit 1
	fi

	echo "Shard ${SHARD}: ${ran} test(s) ran, ${not_passing} not passing, snapshot changes=${HAS_CHANGES:-false}"

	# Upload ONLY the changed files from this shard
	- name: Upload changed snapshots
	uses: actions/upload-artifact@v6
	if: steps.changed-snapshots.outputs.has-changes == 'true'
	with:
	name: snapshots-shard-${{ matrix.shard }}
	path: /tmp/changed_snapshots_shard/
	retention-days: 1

	- name: Upload test report
	uses: actions/upload-artifact@v6
	if: always()
	with:
	name: playwright-report-shard-${{ matrix.shard }}
	path: ./playwright-report/
	retention-days: 30

	# Merge snapshots and commit
	merge-and-commit:
	needs: [setup, update-snapshots-sharded]
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repository
	uses: actions/checkout@v7
	with:
	ref: ${{ needs.setup.outputs.branch }}
	token: ${{ secrets.PR_GH_TOKEN }}

	# Download all changed snapshot files from shards
	- name: Download snapshot artifacts
	uses: actions/download-artifact@v8
	with:
	pattern: snapshots-shard-*
	path: ./downloaded-snapshots
	merge-multiple: true

	- name: List downloaded files
	run: \|
	echo "=========================================="
	echo "DOWNLOADED SNAPSHOT FILES"
	echo "=========================================="
	if [ -d "./downloaded-snapshots" ]; then
	find ./downloaded-snapshots -type f
	echo ""
	echo "Total files: $(find ./downloaded-snapshots -type f \| wc -l)"
	else
	echo "No snapshot artifacts downloaded (no changes in any shard)"
	echo ""
	echo "Total files: 0"
	fi

	# Merge only the changed files into browser_tests
	- name: Merge changed snapshots
	run: \|
	set -euo pipefail

	echo "=========================================="
	echo "MERGING CHANGED SNAPSHOTS"
	echo "=========================================="

	# Check if any artifacts were downloaded (merge-multiple puts files directly in path)
	if [ ! -d "./downloaded-snapshots" ]; then
	echo "No snapshot artifacts to merge"
	echo "=========================================="
	echo "MERGE COMPLETE"
	echo "=========================================="
	echo "Files merged: 0"
	exit 0
	fi

	# Verify target directory exists
	if [ ! -d "browser_tests" ]; then
	echo "::error::Target directory 'browser_tests' does not exist"
	exit 1
	fi

	# Count files to merge
	file_count=$(find ./downloaded-snapshots -type f \| wc -l)

	if [ "$file_count" -eq 0 ]; then
	echo "No snapshot files found in downloaded artifacts"
	echo "=========================================="
	echo "MERGE COMPLETE"
	echo "=========================================="
	echo "Files merged: 0"
	exit 0
	fi

	echo "Merging $file_count snapshot file(s)..."

	# Copy all files directly, preserving directory structure
	# With merge-multiple: true, files are directly in ./downloaded-snapshots/ without shard subdirs
	cp -v -r ./downloaded-snapshots/* browser_tests/ 2>&1 \| sed 's/^/ /'

	echo ""
	echo "=========================================="
	echo "MERGE COMPLETE"
	echo "=========================================="
	echo "Files merged: $file_count"

	- name: Show changes
	run: \|
	echo "=========================================="
	echo "CHANGES SUMMARY"
	echo "=========================================="
	echo ""
	echo "Changed files in browser_tests (including untracked):"
	CHANGES=$(git status --porcelain=v1 --untracked-files=all -- browser_tests/)
	if [ -z "$CHANGES" ]; then
	echo "No changes"
	echo ""
	echo "Total changes:"
	echo "0"
	else
	echo "$CHANGES" \| head -50
	echo ""
	echo "Total changes:"
	echo "$CHANGES" \| wc -l
	fi

	# Flag small rewrites that are likely text-rasterization noise rather than
	# intentional visual changes.
	- name: Report snapshot diff magnitudes
	id: diff-report
	shell: bash
	run: \|
	set -uo pipefail

	report='/tmp/diff-report.md'
	: > "$report"

	mapfile -t changed < <(
	git status --porcelain=v1 --untracked-files=all -- browser_tests/ \|
	sed -e 's/^...//' -e 's/^"//' -e 's/"$//'
	)
	if [ "${#changed[@]}" -eq 0 ]; then
	echo "No snapshot changes to measure"
	exit 0
	fi

	if command -v compare >/dev/null 2>&1; then
	echo "Measuring ${#changed[@]} rewritten snapshot(s) with ImageMagick"
	else
	echo "::warning::ImageMagick 'compare' unavailable; falling back to byte sizes, which cannot distinguish a real visual change from rasterization noise."
	fi

	small=0
	{
	echo "\| snapshot \| status \| changed pixels \|"
	echo "\| --- \| --- \| --- \|"
	} >> "$report"

	tmp_old="$(mktemp)"
	for f in "${changed[@]}"; do
	[ -f "$f" ] \|\| continue
	name="${f#browser_tests/}"

	if ! git show "HEAD:$f" > "$tmp_old" 2>/dev/null; then
	echo "\| \`${name}\` \| new \| n/a \|" >> "$report"
	continue
	fi

	px=''
	if command -v compare >/dev/null 2>&1; then
	px=$(compare -metric AE "$tmp_old" "$f" null: 2>&1 >/dev/null)
	# Non-numeric means differing geometry or a decode failure.
	case "$px" in
	''\|[!0-9]) px='' ;;
	esac
	fi

	if [ -n "$px" ]; then
	echo "\| \`${name}\` \| modified \| ${px} \|" >> "$report"
	# The threshold is advisory because small changes can be valid.
	if [ "$px" -gt 0 ] && [ "$px" -le 2000 ]; then
	small=$((small + 1))
	fi
	else
	old_b=$(wc -c < "$tmp_old" \| tr -d ' ')
	new_b=$(wc -c < "$f" \| tr -d ' ')
	echo "\| \`${name}\` \| modified \| ${old_b}B -> ${new_b}B \|" >> "$report"
	fi
	done
	rm -f "$tmp_old"

	if [ "$small" -gt 0 ]; then
	echo "::warning::${small} rewritten baseline(s) changed by <=2000 pixels. Screenshot capture is not bit-deterministic - small diffs are usually text rasterization noise, not an intended change. Confirm each rewrite corresponds to a test you expected to change before merging."
	fi

	cat "$report"
	{
	echo "### Rewritten snapshots (${#changed[@]})"
	echo
	cat "$report"
	} >> "$GITHUB_STEP_SUMMARY"

	- name: Commit updated expectations
	id: commit
	run: \|
	git config --global user.name 'github-actions'
	git config --global user.email 'github-actions@github.com'

	if [ -z "$(git status --porcelain=v1 --untracked-files=all -- browser_tests/)" ]; then
	echo "No changes to commit"
	echo "has-changes=false" >> $GITHUB_OUTPUT
	exit 0
	fi

	echo "=========================================="
	echo "COMMITTING CHANGES"
	echo "=========================================="

	echo "has-changes=true" >> $GITHUB_OUTPUT

	git add browser_tests/

	# Use cat so backticks and file names remain literal.
	{
	echo '[automated] Update test expectations'
	if [ -s /tmp/diff-report.md ]; then
	echo
	cat /tmp/diff-report.md
	echo
	echo 'Screenshot capture is not bit-deterministic; small pixel'
	echo 'deltas are usually text rasterization noise. Confirm each'
	echo 'rewrite corresponds to a test expected to change.'
	fi
	} > /tmp/commit-msg.txt

	git commit -F /tmp/commit-msg.txt

	echo "Pushing to ${{ needs.setup.outputs.branch }}..."
	git push origin ${{ needs.setup.outputs.branch }}

	echo "✓ Commit and push successful"

	- name: Add Done Reaction
	uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0
	if: github.event_name == 'issue_comment' && steps.commit.outputs.has-changes == 'true'
	with:
	comment-id: ${{ needs.setup.outputs.comment-id }}
	issue-number: ${{ needs.setup.outputs.pr-number }}
	reactions: +1
	reactions-edit-mode: replace

	- name: Remove New Browser Test Expectations label
	if: always() && github.event_name == 'pull_request'
	run: gh pr edit ${{ needs.setup.outputs.pr-number }} --remove-label "New Browser Test Expectations"
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Navigation Menu

[pull] main from Comfy-Org:main #59

Workflow file

[pull] main from Comfy-Org:main #59

Uh oh!

Workflow file for this run