feat: Split GA between jobs #8

	name: ML Pipelines

	on:
	schedule:
	- cron: '0 /2 * *' # Runs every two hours
	push:
	branches:
	- main
	workflow_dispatch: # Allows manual triggering from GitHub UI

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	feature_engineering:
	name: Feature Engineering
	runs-on: ubuntu-latest
	steps:
	- uses: ./.github/actions/setup

	- name: Run pipeline
	run: uv run ipython notebooks/1_feature_engineering.ipynb
	env:
	HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}

	train_retrieval:
	needs: feature_engineering
	name: Train Retrieval Model
	runs-on: ubuntu-latest
	steps:
	- uses: ./.github/actions/setup

	- name: Run pipeline
	run: uv run ipython notebooks/2_train_retrieval_model.ipynb
	env:
	HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}

	create_embeddings:
	needs: train_retrieval
	name: Create Embeddings
	runs-on: ubuntu-latest
	steps:
	- uses: ./.github/actions/setup

	- name: Run pipeline
	run: uv run ipython notebooks/3_embeddings_creation.ipynb
	env:
	HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}

	train_ranking:
	needs: create_embeddings
	name: Train Ranking Model
	runs-on: ubuntu-latest
	steps:
	- uses: ./.github/actions/setup

	- name: Run pipeline
	run: uv run ipython notebooks/4_train_ranking_model.ipynb
	env:
	HOPSWORKS_API_KEY: ${{ secrets.HOPSWORKS_API_KEY }}

Provide feedback