Skip to content

Commit f0bc381

Browse files
authored
Improve handling of parallel repo management (#155)
* Implement new parallel repository workflow in GitHub Actions Updated infrastructure-repository-update.yml to use the new 3-step parallel repository architecture that eliminates duplicate work. Workflow changes: - Step 1: main job builds common component with update-main - Step 2: postclean jobs build release-specific components in parallel using -R flag (isolated DBs, no publishing) - Step 3: NEW merge job combines common + release-specific components and publishes complete repositories - Updated cleanup job dependency to wait for merge Benefits: - Common packages added once instead of per-release - True parallelism across multiple releases - Better error isolation between releases - Scalable architecture for adding more releases Signed-off-by: Igor Pecovnik <igor@armbian.com> * Use dev branch * Revert "Use dev branch" This reverts commit 7f06a08. --------- Signed-off-by: Igor Pecovnik <igor@armbian.com>
1 parent d59abd6 commit f0bc381

1 file changed

Lines changed: 102 additions & 3 deletions

File tree

.github/workflows/infrastructure-repository-update.yml

Lines changed: 102 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -581,9 +581,10 @@ jobs:
581581
INPUT_DIR="${STORAGE_PATH}/${PACKAGE}"
582582
OUTPUT_DIR="${PUBLISH_PATH}-${PACKAGE}"
583583
584-
echo "### Checking for .deb files in $PACKAGE/$RELEASE" | tee -a "$GITHUB_STEP_SUMMARY"
584+
echo "### Building release-specific components for $PACKAGE/$RELEASE" | tee -a "$GITHUB_STEP_SUMMARY"
585585
echo ""
586586
echo "Input directory: $INPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY"
587+
echo "Output directory: $OUTPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY"
587588
echo ""
588589
589590
# Validate input directory exists
@@ -599,20 +600,118 @@ jobs:
599600
600601
# Only run repo.sh if there are files to process
601602
if [ "${DEB_COUNT}" -gt 0 ]; then
602-
echo "Running repository update for $RELEASE..." | tee -a "$GITHUB_STEP_SUMMARY"
603+
echo "Building release-specific repos and snapshots for $RELEASE..." | tee -a "$GITHUB_STEP_SUMMARY"
604+
echo "Note: Common component was already built by 'main' job" | tee -a "$GITHUB_STEP_SUMMARY"
605+
echo "Release-specific components will be merged by 'merge' job" | tee -a "$GITHUB_STEP_SUMMARY"
606+
echo ""
603607
tools/repository/repo.sh \
604608
-c update \
605609
-R "$RELEASE" \
606610
-k \
607611
-i "$INPUT_DIR" \
608612
-o "$OUTPUT_DIR"
613+
echo "✓ Release-specific components built for $RELEASE" | tee -a "$GITHUB_STEP_SUMMARY"
609614
else
610615
echo "::notice::No .deb files found, skipping repository update"
611616
fi
612617
618+
merge:
619+
needs: postclean
620+
strategy:
621+
fail-fast: false
622+
matrix:
623+
repository:
624+
- name: debs
625+
- name: debs-beta
626+
name: "Merge ${{ matrix.repository.name }}"
627+
timeout-minutes: 120
628+
runs-on: repository
629+
steps:
630+
631+
# Cleaning self hosted runners
632+
- name: Runner clean
633+
uses: armbian/actions/runner-clean@main
634+
635+
- name: Import GPG key
636+
uses: crazy-max/ghaction-import-gpg@v6
637+
with:
638+
gpg_private_key: ${{ secrets.GPG_KEY3 }}
639+
640+
- name: Import GPG key
641+
uses: crazy-max/ghaction-import-gpg@v6
642+
with:
643+
gpg_private_key: ${{ secrets.GPG_KEY4 }}
644+
645+
- name: Checkout build repository
646+
uses: actions/checkout@v6
647+
with:
648+
repository: armbian/build
649+
ref: main
650+
fetch-depth: 1
651+
clean: false
652+
653+
- name: "Merge repository ${{ matrix.repository.name }}"
654+
shell: bash
655+
run: |
656+
set -e
657+
set -o pipefail
658+
659+
STORAGE_PATH="${{ env.STORAGE_PATH }}"
660+
PUBLISH_PATH="${{ env.PUBLISHING_PATH }}"
661+
REPO_NAME="${{ matrix.repository.name }}"
662+
663+
# Validate paths
664+
case "$STORAGE_PATH" in
665+
*"/armbian/openssh-server/storage"*)
666+
;;
667+
*)
668+
echo "::error::Invalid storage path: $STORAGE_PATH"
669+
exit 1
670+
;;
671+
esac
672+
673+
case "$PUBLISH_PATH" in
674+
*"/publishing/repository"*)
675+
;;
676+
*)
677+
echo "::error::Invalid publishing path: $PUBLISH_PATH"
678+
exit 1
679+
;;
680+
esac
681+
682+
INPUT_DIR="${STORAGE_PATH}/${REPO_NAME}"
683+
OUTPUT_DIR="${PUBLISH_PATH}-${REPO_NAME}"
684+
685+
echo "### Merging repository $REPO_NAME" | tee -a "$GITHUB_STEP_SUMMARY"
686+
echo ""
687+
echo "Input directory: $INPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY"
688+
echo "Output directory: $OUTPUT_DIR" | tee -a "$GITHUB_STEP_SUMMARY"
689+
echo ""
690+
691+
# Validate output directory exists
692+
if [ ! -d "$OUTPUT_DIR" ]; then
693+
echo "::notice::Output directory does not exist: $OUTPUT_DIR"
694+
echo "This is expected if no packages were processed yet" | tee -a "$GITHUB_STEP_SUMMARY"
695+
exit 0
696+
fi
697+
698+
echo "Merging common component with release-specific components..." | tee -a "$GITHUB_STEP_SUMMARY"
699+
echo "This imports from isolated databases and publishes complete repositories" | tee -a "$GITHUB_STEP_SUMMARY"
700+
echo ""
701+
702+
# Run merge command to combine common + release-specific components
703+
tools/repository/repo.sh \
704+
-c merge \
705+
-i "$INPUT_DIR" \
706+
-o "$OUTPUT_DIR"
707+
708+
echo "✓ Repository merge completed for $REPO_NAME" | tee -a "$GITHUB_STEP_SUMMARY"
709+
echo ""
710+
echo "Repository is now ready with all components combined" | tee -a "$GITHUB_STEP_SUMMARY"
711+
613712
cleanup:
614713
name: "Clean input"
615-
needs: postclean
714+
needs: merge
616715
runs-on: repository
617716
strategy:
618717
matrix:

0 commit comments

Comments
 (0)