Daily Update #42

Workflow file for this run

.github/workflows/daily-update.yml at 9d0abec

	# Daily automated pipeline: Fetch data → Calculate TOTAL2 → Deploy to GitHub Pages
	# Runs every day at 6:00 AM UTC
	name: Daily Update

	on:
	# Scheduled trigger - daily at 6:00 AM UTC
	schedule:
	- cron: '0 6 * * *'

	# Manual trigger for testing
	workflow_dispatch:
	inputs:
	skip_fetch:
	description: 'Skip fetching new data (use existing raw-data)'
	required: false
	default: false
	type: boolean

	# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
	permissions:
	contents: write
	pages: write
	id-token: write

	# Allow only one concurrent run
	concurrency:
	group: "daily-update"
	cancel-in-progress: false

	env:
	INDEX_TYPE: 'total2b'

	jobs:
	# ==========================================================================
	# Step 1: Fetch Raw Data
	# ==========================================================================
	fetch-data:
	runs-on: ubuntu-latest
	# Skip if manually triggered with skip_fetch=true
	if: ${{ github.event.inputs.skip_fetch != 'true' }}

	steps:
	- name: Checkout main branch
	uses: actions/checkout@v4
	with:
	path: main

	- name: Checkout raw-data branch
	id: checkout-raw-data
	uses: actions/checkout@v4
	with:
	ref: raw-data
	path: raw-data-branch
	continue-on-error: true

	- name: Initialize raw-data branch if missing
	if: steps.checkout-raw-data.outcome == 'failure'
	run: \|
	mkdir -p raw-data-branch
	cd raw-data-branch
	git init
	git checkout --orphan raw-data
	echo "# Halvix Raw Data" > README.md
	echo "" >> README.md
	echo "This branch contains raw price data (parquet files)." >> README.md
	echo "It is automatically updated by CI and has no history (orphan branch)." >> README.md
	echo "" >> README.md
	echo "Do not manually edit files in this branch." >> README.md
	git add README.md
	git config user.name "github-actions[bot]"
	git config user.email "github-actions[bot]@users.noreply.github.com"
	git commit -m "Initialize raw-data branch"
	echo "Initialized new orphan raw-data branch"

	- name: Restore cached data to main workspace
	run: \|
	if [ -d "raw-data-branch/raw" ]; then
	mkdir -p main/data
	cp -r raw-data-branch/raw main/data/ 2>/dev/null \|\| true
	cp -r raw-data-branch/cache main/data/ 2>/dev/null \|\| true
	# Copy processed files (coins list, metadata, skipped/failed)
	mkdir -p main/data/processed
	cp raw-data-branch/processed/coins_to_download.json main/data/processed/ 2>/dev/null \|\| true
	cp raw-data-branch/processed/download_skipped.csv main/data/processed/ 2>/dev/null \|\| true
	cp raw-data-branch/processed/download_failed.csv main/data/processed/ 2>/dev/null \|\| true
	cp raw-data-branch/processed/no_usd_data.csv main/data/processed/ 2>/dev/null \|\| true
	cp raw-data-branch/processed/fetch_metadata.json main/data/processed/ 2>/dev/null \|\| true
	echo "Restored cached data from raw-data branch"
	echo "Price files found:"
	ls -la main/data/raw/prices/ 2>/dev/null \| head -20 \|\| echo "No price files yet"
	else
	echo "No existing data in raw-data branch (first run)"
	mkdir -p main/data/raw/prices
	mkdir -p main/data/processed
	mkdir -p main/data/cache
	fi

	- name: Set up Python 3.13
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: latest
	virtualenvs-create: true
	virtualenvs-in-project: true

	- name: Load cached venv
	id: cached-poetry-dependencies
	uses: actions/cache@v4
	with:
	path: main/.venv
	key: venv-daily-${{ runner.os }}-3.13-${{ hashFiles('main/**/pyproject.toml') }}

	- name: Install dependencies
	working-directory: main
	run: poetry install --no-interaction

	- name: Fetch coin list
	working-directory: main
	run: poetry run python -m main list-coins --skip-ping

	- name: Fetch price data (incremental)
	working-directory: main
	run: poetry run python -m main fetch-prices

	- name: Show data status
	working-directory: main
	run: poetry run python -m main status

	- name: Update raw-data branch with new data
	run: \|
	cd raw-data-branch

	git config user.name "github-actions[bot]"
	git config user.email "github-actions[bot]@users.noreply.github.com"

	find . -maxdepth 1 -type d ! -name '.' ! -name '.git' -exec rm -rf {} + 2>/dev/null \|\| true

	if [ -d "../main/data/raw" ]; then
	cp -r ../main/data/raw . 2>/dev/null \|\| true
	echo "Copied raw data"
	fi
	if [ -d "../main/data/cache" ]; then
	cp -r ../main/data/cache . 2>/dev/null \|\| true
	echo "Copied cache data"
	fi
	# Copy processed files (coins list, metadata, skipped/failed, no-USD)
	mkdir -p processed
	for file in coins_to_download.json download_skipped.csv download_failed.csv no_usd_data.csv fetch_metadata.json; do
	if [ -f "../main/data/processed/$file" ]; then
	cp "../main/data/processed/$file" processed/ 2>/dev/null \|\| true
	fi
	done

	# Copy data_status.html (generated by list-coins/fetch-prices)
	mkdir -p site
	if [ -f "../main/site/data_status.html" ]; then
	cp ../main/site/data_status.html site/ 2>/dev/null \|\| true
	echo "Copied data_status.html"
	fi

	echo "# Halvix Raw Data" > README.md
	echo "" >> README.md
	echo "This branch contains raw price data (parquet files)." >> README.md
	echo "It is automatically updated by CI and has no history (orphan branch)." >> README.md
	echo "" >> README.md
	echo "Last updated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> README.md
	echo "" >> README.md
	echo "Do not manually edit files in this branch." >> README.md
	echo "" >> README.md
	echo "## Contents" >> README.md
	echo "" >> README.md

	if [ -d "raw/prices" ]; then
	count=$(ls raw/prices/*.parquet 2>/dev/null \| wc -l)
	echo "- \`raw/prices/\`: $count coin price files" >> README.md
	fi
	if [ -d "cache" ]; then
	echo "- \`cache/\`: Coin list cache" >> README.md
	fi
	if [ -d "processed" ]; then
	echo "- \`processed/\`: Download metadata (coins list, fetch metadata, skipped/failed)" >> README.md
	fi
	if [ -d "site" ]; then
	echo "- \`site/\`: Generated HTML pages (data_status.html)" >> README.md
	fi

	echo "=== Files to commit ==="
	ls -la
	echo ""

	git add -A

	if git diff --staged --quiet; then
	echo "No changes to commit"
	exit 0
	fi

	echo "=== Staged changes ==="
	git diff --staged --stat
	echo ""

	if git rev-parse HEAD >/dev/null 2>&1; then
	echo "Amending existing commit to squash history..."
	git commit --amend -m "Update raw data - $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	else
	echo "Creating initial commit..."
	git commit -m "Update raw data - $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	fi

	git remote set-url origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" 2>/dev/null \|\| \
	git remote add origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" 2>/dev/null \|\| true

	echo "Pushing to raw-data branch..."
	git push --force origin raw-data

	echo "=== Raw data branch updated successfully ==="

	# ==========================================================================
	# Step 2: Calculate TOTAL2 Index
	# ==========================================================================
	calculate:
	runs-on: ubuntu-latest
	needs: [fetch-data]
	# Run even if fetch-data was skipped (when skip_fetch=true)
	if: ${{ always() && (needs.fetch-data.result == 'success' \|\| needs.fetch-data.result == 'skipped') }}

	steps:
	- name: Checkout main branch
	uses: actions/checkout@v4
	with:
	path: main

	- name: Checkout raw-data branch
	id: checkout-raw-data
	uses: actions/checkout@v4
	with:
	ref: raw-data
	path: raw-data-branch
	continue-on-error: true

	- name: Check raw-data branch exists
	if: steps.checkout-raw-data.outcome == 'failure'
	run: \|
	echo "❌ Error: raw-data branch does not exist."
	echo "Please run the 'Fetch Raw Data' workflow first."
	exit 1

	- name: Checkout processed-data branch
	id: checkout-processed-data
	uses: actions/checkout@v4
	with:
	ref: processed-data
	path: processed-data-branch
	continue-on-error: true

	- name: Initialize processed-data branch if missing
	if: steps.checkout-processed-data.outcome == 'failure'
	run: \|
	mkdir -p processed-data-branch
	cd processed-data-branch
	git init
	git checkout --orphan processed-data
	echo "# Halvix Processed Data" > README.md
	echo "" >> README.md
	echo "This branch contains processed TOTAL2 index data." >> README.md
	echo "It is automatically updated by CI and has no history (orphan branch)." >> README.md
	echo "" >> README.md
	echo "Do not manually edit files in this branch." >> README.md
	git add README.md
	git config user.name "github-actions[bot]"
	git config user.email "github-actions[bot]@users.noreply.github.com"
	git commit -m "Initialize processed-data branch"
	echo "Initialized new orphan processed-data branch"

	- name: Restore data to main workspace
	run: \|
	mkdir -p main/data/raw
	mkdir -p main/data/processed
	mkdir -p main/data/cache

	if [ -d "raw-data-branch/raw" ]; then
	cp -r raw-data-branch/raw/* main/data/raw/ 2>/dev/null \|\| true
	echo "Restored raw data from raw-data branch"
	fi
	if [ -d "raw-data-branch/cache" ]; then
	cp -r raw-data-branch/cache/* main/data/cache/ 2>/dev/null \|\| true
	echo "Restored cache from raw-data branch"
	fi

	if [ -d "processed-data-branch/processed" ]; then
	cp -r processed-data-branch/processed/* main/data/processed/ 2>/dev/null \|\| true
	echo "Restored existing processed data from processed-data branch"
	fi

	echo "Price files found:"
	ls -la main/data/raw/prices/ 2>/dev/null \| head -20 \|\| echo "No price files"

	- name: Set up Python 3.13
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: latest
	virtualenvs-create: true
	virtualenvs-in-project: true

	- name: Load cached venv
	id: cached-poetry-dependencies
	uses: actions/cache@v4
	with:
	path: main/.venv
	key: venv-daily-${{ runner.os }}-3.13-${{ hashFiles('main/**/pyproject.toml') }}

	- name: Install dependencies
	working-directory: main
	run: poetry install --no-interaction

	- name: Calculate TOTAL2 index
	working-directory: main
	run: \|
	poetry run python -m main calculate-total2 --index-type ${{ env.INDEX_TYPE }}

	- name: Show data status
	working-directory: main
	run: poetry run python -m main status

	- name: Update processed-data branch
	run: \|
	cd processed-data-branch

	git config user.name "github-actions[bot]"
	git config user.email "github-actions[bot]@users.noreply.github.com"

	find . -maxdepth 1 -type d ! -name '.' ! -name '.git' -exec rm -rf {} + 2>/dev/null \|\| true

	mkdir -p processed
	if [ -f "../main/data/processed/total2_index.parquet" ]; then
	cp ../main/data/processed/total2_index.parquet processed/
	echo "Copied total2_index.parquet"
	fi
	if [ -f "../main/data/processed/total2_daily_composition.parquet" ]; then
	cp ../main/data/processed/total2_daily_composition.parquet processed/
	echo "Copied total2_daily_composition.parquet"
	fi
	if [ -f "../main/data/processed/total2_max_weight_change.json" ]; then
	cp ../main/data/processed/total2_max_weight_change.json processed/
	echo "Copied total2_max_weight_change.json"
	fi

	echo "# Halvix Processed Data" > README.md
	echo "" >> README.md
	echo "This branch contains processed TOTAL2 index data." >> README.md
	echo "It is automatically updated by CI and has no history (orphan branch)." >> README.md
	echo "" >> README.md
	echo "Last updated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> README.md
	echo "Index type: ${{ env.INDEX_TYPE }}" >> README.md
	echo "" >> README.md
	echo "Do not manually edit files in this branch." >> README.md
	echo "" >> README.md
	echo "## Contents" >> README.md
	echo "" >> README.md
	echo "- \`processed/total2_index.parquet\`: TOTAL2 index time series" >> README.md
	echo "- \`processed/total2_daily_composition.parquet\`: Daily coin composition" >> README.md
	echo "- \`processed/total2_max_weight_change.json\`: Statistics and metadata" >> README.md

	echo "=== Files to commit ==="
	ls -la
	ls -la processed/ 2>/dev/null \|\| true
	echo ""

	git add -A

	if git diff --staged --quiet; then
	echo "No changes to commit"
	exit 0
	fi

	echo "=== Staged changes ==="
	git diff --staged --stat
	echo ""

	if git rev-parse HEAD >/dev/null 2>&1; then
	echo "Amending existing commit to squash history..."
	git commit --amend -m "Update TOTAL2 index (${{ env.INDEX_TYPE }}) - $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	else
	echo "Creating initial commit..."
	git commit -m "Update TOTAL2 index (${{ env.INDEX_TYPE }}) - $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
	fi

	git remote set-url origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" 2>/dev/null \|\| \
	git remote add origin "https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git" 2>/dev/null \|\| true

	echo "Pushing to processed-data branch..."
	git push --force origin processed-data

	echo "=== Processed data branch updated successfully ==="

	# ==========================================================================
	# Step 3: Generate Charts and Deploy to GitHub Pages
	# ==========================================================================
	build:
	runs-on: ubuntu-latest
	needs: [calculate]
	if: ${{ always() && needs.calculate.result == 'success' }}

	steps:
	- name: Checkout main branch
	uses: actions/checkout@v4
	with:
	path: main

	- name: Checkout raw-data branch
	id: checkout-raw-data
	uses: actions/checkout@v4
	with:
	ref: raw-data
	path: raw-data-branch
	continue-on-error: true

	- name: Check raw-data branch exists
	if: steps.checkout-raw-data.outcome == 'failure'
	run: \|
	echo "⚠️ Warning: raw-data branch does not exist."
	echo "Charts requiring price data may fail."

	- name: Checkout processed-data branch
	id: checkout-processed-data
	uses: actions/checkout@v4
	with:
	ref: processed-data
	path: processed-data-branch
	continue-on-error: true

	- name: Check processed-data branch exists
	if: steps.checkout-processed-data.outcome == 'failure'
	run: \|
	echo "❌ Error: processed-data branch does not exist."
	echo "Please run the 'Calculate TOTAL2' workflow first."
	exit 1

	- name: Restore data to main workspace
	run: \|
	mkdir -p main/data/raw
	mkdir -p main/data/processed
	mkdir -p main/data/cache

	if [ -d "raw-data-branch/raw" ]; then
	cp -r raw-data-branch/raw/* main/data/raw/ 2>/dev/null \|\| true
	echo "Restored raw data from raw-data branch"
	fi
	if [ -d "raw-data-branch/cache" ]; then
	cp -r raw-data-branch/cache/* main/data/cache/ 2>/dev/null \|\| true
	echo "Restored cache from raw-data branch"
	fi

	if [ -d "processed-data-branch/processed" ]; then
	cp -r processed-data-branch/processed/* main/data/processed/ 2>/dev/null \|\| true
	echo "Restored processed data from processed-data branch"
	fi

	# Restore data_status.html (generated by list-coins/fetch-prices)
	if [ -f "raw-data-branch/site/data_status.html" ]; then
	mkdir -p main/site
	cp raw-data-branch/site/data_status.html main/site/ 2>/dev/null \|\| true
	echo "Restored data_status.html from raw-data branch"
	fi

	echo "Data files restored:"
	ls -la main/data/raw/prices/ 2>/dev/null \| head -10 \|\| echo "No price files"
	ls -la main/data/processed/ 2>/dev/null \|\| echo "No processed files"

	- name: Set up Python 3.13
	uses: actions/setup-python@v5
	with:
	python-version: "3.13"

	- name: Install Poetry
	uses: snok/install-poetry@v1
	with:
	version: latest
	virtualenvs-create: true
	virtualenvs-in-project: true

	- name: Load cached venv
	id: cached-poetry-dependencies
	uses: actions/cache@v4
	with:
	path: main/.venv
	key: venv-daily-${{ runner.os }}-3.13-${{ hashFiles('main/**/pyproject.toml') }}

	- name: Install dependencies
	working-directory: main
	run: poetry install --no-interaction

	- name: Generate charts
	working-directory: main
	run: \|
	poetry run python -m main generate-charts
	echo "Charts generated in site/"
	ls -la site/charts/ \| head -20

	- name: Setup Pages
	uses: actions/configure-pages@v5

	- name: Upload artifact
	uses: actions/upload-pages-artifact@v3
	with:
	path: "main/site"

	# Deployment job
	deploy:
	environment:
	name: github-pages
	url: ${{ steps.deployment.outputs.page_url }}
	runs-on: ubuntu-latest
	needs: build
	if: ${{ always() && needs.build.result == 'success' }}
	steps:
	- name: Deploy to GitHub Pages
	id: deployment
	uses: actions/deploy-pages@v4

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Daily Update #42

Workflow file

Daily Update #42

Uh oh!

Workflow file for this run