Imaging RFC Implementation #17

Workflow file for this run

.github/workflows/generate-python-classes.yml at d2daa1e

	name: Generate Python Classes

	on:
	push:
	branches: [ main ]
	paths:
	- 'modules//domains/.yaml'
	- 'modules//src//.py'
	- 'modules/*/Makefile'
	- 'Makefile'
	- 'config/config.yaml'
	pull_request:
	branches: [ main ]
	paths:
	- 'modules//domains/.yaml'
	- 'modules//src//.py'
	- 'modules/*/Makefile'
	- 'Makefile'
	- 'config/config.yaml'
	- '.github/workflows/generate-python-classes.yml'
	workflow_dispatch:
	inputs:
	modules:
	description: 'Modules to regenerate (comma-separated: biospecimen,clinical,wes,sequencing,imaging,scrna-seq,digitalpathology,multiplexmicroscopy,all). Leave empty or "all" to generate all modules.'
	required: false
	default: 'all'
	type: choice
	options:
	- all
	- biospecimen
	- clinical
	- wes
	- sequencing
	- imaging
	- scrna-seq
	- digitalpathology
	- multiplexmicroscopy
	- biospecimen,clinical
	- biospecimen,wes
	- clinical,wes

	jobs:
	generate-python-classes:
	runs-on: ubuntu-latest

	# Grant permissions to write contents (for auto-committing to PR branches)
	permissions:
	contents: write
	pull-requests: write

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	fetch-depth: 0 # Required for pushing to PR branches
	# actions/checkout@v4 automatically checks out PR branch for PR events

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: latest
	virtualenvs-create: true
	virtualenvs-in-project: true

	- name: Load cached venv
	id: cached-poetry-dependencies
	uses: actions/cache@v3
	with:
	path: .venv
	key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

	- name: Install dependencies
	if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
	run: poetry install --no-interaction --no-root

	- name: Install project
	run: poetry install --no-interaction

	# Determine which modules to generate
	# For automatic triggers (push/PR), generate all modules
	# For manual dispatch, use the input or default to all
	- name: Set modules to generate
	id: set-modules
	run: \|
	if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
	MODULES="${{ github.event.inputs.modules }}"
	if [ -z "$MODULES" ] \|\| [ "$MODULES" = "all" ]; then
	echo "modules=all" >> $GITHUB_OUTPUT
	echo "Generating all modules (manual trigger)"
	else
	echo "modules=$MODULES" >> $GITHUB_OUTPUT
	echo "Generating selected modules: $MODULES"
	fi
	else
	echo "modules=all" >> $GITHUB_OUTPUT
	echo "Generating all modules (automatic trigger)"
	fi

	# Generate all modules (for automatic triggers or when "all" is selected)
	- name: Generate Python Classes for All Modules
	if: steps.set-modules.outputs.modules == 'all'
	run: make modules-gen

	# Generate selected modules individually (for manual dispatch with specific selection)
	- name: Generate Python Classes for Biospecimen
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "biospecimen"; then
	echo "🔄 Generating Python classes for Biospecimen module..."
	cd modules/Biospecimen
	make gen-schema
	echo "✅ Biospecimen Python classes generated"
	else
	echo "⏭️ Skipping Biospecimen (not in selected modules)"
	fi

	- name: Generate Python Classes for Clinical
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "clinical"; then
	echo "🔄 Generating Python classes for Clinical module..."
	cd modules/Clinical
	make gen-schema
	echo "✅ Clinical Python classes generated"
	else
	echo "⏭️ Skipping Clinical (not in selected modules)"
	fi

	- name: Generate Python Classes for WES
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "wes"; then
	echo "🔄 Generating Python classes for WES module..."
	cd modules/WES
	make gen-schema
	echo "✅ WES Python classes generated"
	else
	echo "⏭️ Skipping WES (not in selected modules)"
	fi

	- name: Generate Python Classes for Sequencing
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "sequencing"; then
	echo "🔄 Generating Python classes for Sequencing module..."
	cd modules/Sequencing
	make gen-schema
	echo "✅ Sequencing Python classes generated"
	else
	echo "⏭️ Skipping Sequencing (not in selected modules)"
	fi

	- name: Generate Python Classes for Imaging
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "imaging"; then
	echo "🔄 Generating Python classes for Imaging module..."
	cd modules/Imaging
	make gen-schema
	echo "✅ Imaging Python classes generated"
	else
	echo "⏭️ Skipping Imaging (not in selected modules)"
	fi

	- name: Generate Python Classes for scRNA-seq
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "scrna-seq"; then
	echo "🔄 Generating Python classes for scRNA-seq module..."
	cd modules/scRNA-seq
	make gen-schema
	echo "✅ scRNA-seq Python classes generated"
	else
	echo "⏭️ Skipping scRNA-seq (not in selected modules)"
	fi

	- name: Generate Python Classes for DigitalPathology
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "digitalpathology"; then
	echo "🔄 Generating Python classes for DigitalPathology module..."
	cd modules/DigitalPathology
	make gen-schema
	echo "✅ DigitalPathology Python classes generated"
	else
	echo "⏭️ Skipping DigitalPathology (not in selected modules)"
	fi

	- name: Generate Python Classes for MultiplexMicroscopy
	if: steps.set-modules.outputs.modules != 'all'
	run: \|
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if echo "$MODULES" \| grep -q "multiplexmicroscopy"; then
	echo "🔄 Generating Python classes for MultiplexMicroscopy module..."
	cd modules/MultiplexMicroscopy
	make gen-schema
	echo "✅ MultiplexMicroscopy Python classes generated"
	else
	echo "⏭️ Skipping MultiplexMicroscopy (not in selected modules)"
	fi

	- name: Run Tests
	id: run-tests
	continue-on-error: true
	run: \|
	poetry run pytest modules/*/tests/ -v --tb=short

	- name: Check for Changes
	if: steps.run-tests.outcome == 'success'
	id: check-changes
	run: \|
	if git diff --quiet; then
	echo "✅ No changes detected - Python classes are up to date"
	echo "has-changes=false" >> $GITHUB_OUTPUT
	else
	echo "❌ Changes detected in generated Python classes"
	echo "has-changes=true" >> $GITHUB_OUTPUT
	echo "📋 Changed files:"
	git diff --name-only
	fi

	- name: Auto-commit generated classes to PR branch
	id: auto-commit
	if: steps.run-tests.outcome == 'success' && steps.check-changes.outputs.has-changes == 'true' && github.event_name == 'pull_request'
	run: \|
	# NOTE: For PR events, we push directly to the same PR branch (no new PR created)
	# For push to main events, we fail the workflow if classes are outdated (safety check)

	# Configure git
	git config --local user.email "action@github.com"
	git config --local user.name "GitHub Action"

	# Get the PR branch name
	PR_BRANCH="${{ github.event.pull_request.head.ref }}"
	PR_REPO="${{ github.event.pull_request.head.repo.full_name }}"

	# Check if this is a same-repo PR (GITHUB_TOKEN can push) or fork PR (needs PAT/GitHub App)
	if [ "$PR_REPO" != "${{ github.repository }}" ]; then
	echo "⚠️ This is a fork PR. Auto-commit requires a GitHub App or PAT token."
	echo "Please run 'make modules-gen' locally and push the changes."
	exit 0
	fi

	# Add generated Python classes
	git add modules//src/htan_/datamodel/*.py \|\| true

	# Capture list of changed files before committing (for PR comment)
	CHANGED_FILES=$(git diff --cached --name-only \|\| echo "")
	# Use multiline output format for GitHub Actions
	{
	echo "changed_files<<EOF"
	echo "$CHANGED_FILES"
	echo "EOF"
	} >> $GITHUB_OUTPUT

	# Commit changes (skip if no changes or commit already exists)
	git commit -m "chore: auto-generate Python classes from schema changes

	Auto-generated Python classes from LinkML schema updates.

	Auto-generated by GitHub Actions workflow
	[skip ci]" \|\| echo "No changes to commit or commit already exists"

	# Push to the same PR branch (updates the existing PR, no new PR created)
	git push origin "HEAD:$PR_BRANCH" \|\| echo "Push failed"

	- name: Create Pull Request (manual dispatch only)
	if: steps.run-tests.outcome == 'success' && steps.check-changes.outputs.has-changes == 'true' && github.event_name == 'workflow_dispatch'
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	# Configure git
	git config --local user.email "action@github.com"
	git config --local user.name "GitHub Action"

	# Create a new branch for this update
	branch_name="update-python-classes-$(date +%Y%m%d-%H%M%S)"
	git checkout -b "$branch_name"

	# Add all changes
	git add modules//src/htan_/datamodel/*.py

	# Get list of modules that were generated
	MODULES="${{ steps.set-modules.outputs.modules }}"
	if [ "$MODULES" = "all" ]; then
	changed_modules="all modules"
	else
	changed_modules=$(echo "$MODULES" \| tr ',' ' ')
	fi

	# Commit changes
	git commit -m "feat: update Python classes for modules: $changed_modules

	Auto-generated Python classes from LinkML schema updates.

	### Updated Modules:
	$changed_modules

	### Changes:
	- Updated datamodel classes using Makefile gen-schema targets
	- All tests passing

	Manually triggered Python class generation"

	# Push branch
	git push origin "$branch_name"

	# Create pull request
	gh pr create \
	--title "Update Python Classes for $changed_modules" \
	--body "## Python Class Generation Update

	This PR updates the Python data model classes for the following modules:
	$changed_modules

	### Generated Classes:
	- Regenerated all Python datamodel classes using Makefile gen-schema targets
	- All tests are passing ✅

	### Testing:
	- All module tests pass
	- Generated classes are syntactically valid
	- Schema compatibility maintained

	Manually triggered by GitHub Actions workflow" \
	--base main \
	--head "$branch_name"

	- name: Fail if Python classes are outdated (push to main only)
	if: steps.run-tests.outcome == 'success' && steps.check-changes.outputs.has-changes == 'true' && github.event_name == 'push'
	run: \|
	echo "❌ DEPLOYMENT CHECK FAILED"
	echo ""
	echo "The Python classes are not up to date with the current schemas."
	echo "This means the generated Python code doesn't match the LinkML YAML files."
	echo ""
	echo "To fix this:"
	echo "1. Manually trigger this workflow (workflow_dispatch) to regenerate classes"
	echo "2. Or run locally: \`make modules-gen\`"
	echo "3. Commit the updated Python classes"
	echo ""
	echo "Changed files:"
	git diff --name-only
	exit 1

	- name: Fail if tests failed
	if: steps.run-tests.outcome == 'failure'
	run: \|
	echo "❌ TESTS FAILED - Python class generation skipped"
	echo ""
	echo "Tests must pass before Python classes can be generated and committed."
	echo "Please fix the failing tests and try again."
	exit 1

	- name: Comment on PR with Status
	if: steps.run-tests.outcome == 'success' && github.event_name == 'pull_request'
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	if [ "${{ steps.check-changes.outputs.has-changes }}" == "true" ]; then
	# Use the captured list of changed files from the auto-commit step
	# If not available, fall back to showing files from the most recent commit
	if [ -n "${{ steps.auto-commit.outputs.changed_files }}" ]; then
	CHANGED_FILES="${{ steps.auto-commit.outputs.changed_files }}"
	else
	CHANGED_FILES=$(git show --name-only --pretty="" HEAD \|\| echo "")
	fi

	gh pr comment ${{ github.event.pull_request.number }} \
	--body "✅ Python classes auto-updated!

	The Python classes have been automatically regenerated and committed to this PR branch.

	Updated files:
	\`\`\`
	$CHANGED_FILES
	\`\`\`

	The generated classes are now up to date with the schema changes."
	else
	gh pr comment ${{ github.event.pull_request.number }} \
	--body "✅ Python classes are up to date!

	All Python classes match the current schemas. No updates needed."
	fi

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Imaging RFC Implementation #17

Workflow file

Imaging RFC Implementation #17

Uh oh!

Workflow file for this run