Soak Testing for RPI5 Led-Blinker #3954
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,207 @@ | ||
| # Cross-compile https://github.com/moisesmata/fprime-workshop-led-blinker (add/comLogger-instance branch) as system service on AArch64 Linux | ||
| # Builds on release publication, stops old service, starts new service | ||
| # | ||
| # This workflow sets up two types of logging: | ||
| # 1. GDS logs: Text-based logs from fprime-gds → /opt/ledblinker-soak/logs/ | ||
| # 2. ComLogger logs: Binary .com files from ComLogger component → /opt/ledblinker-soak/ComLoggerFiles/ | ||
| # | ||
| name: "Soak Test Setup AArch64 Linux: External Repo LedBlinker" | ||
|
|
||
| on: | ||
| release: | ||
| types: [published] | ||
| workflow_dispatch: | ||
| inputs: | ||
| service_user: | ||
| description: "Linux user to run services as (default: fprime)" | ||
| required: false | ||
| default: fprime | ||
|
|
||
| # New releases cancel running deployments | ||
| concurrency: | ||
| group: ${{ github.workflow }} | ||
| cancel-in-progress: true | ||
|
|
||
| env: | ||
| AARCH64_TOOLCHAIN_DIR: /tmp/aarch64-toolchain | ||
| AARCH64_TOOLCHAIN_URL: https://developer.arm.com/-/media/Files/downloads/gnu-a/10.2-2020.11/binrel/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu.tar.xz | ||
| FSW_SERVICE_NAME: ledblinker-soak-fsw | ||
| GDS_SERVICE_NAME: ledblinker-soak-gds | ||
| INSTALL_DIR: /opt/ledblinker-soak | ||
| SERVICE_USER: ${{ github.event.inputs.service_user || 'fprime' }} | ||
|
|
||
| jobs: | ||
| cross-compilation: | ||
| name: "Cross Compilation" | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: "Checkout target repository" | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| submodules: false | ||
| repository: moisesmata/fprime-workshop-led-blinker | ||
| ref: add/comLogger-instance | ||
| - name: "Overlay current F´ revision" | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| submodules: true | ||
| path: ./fprime | ||
| fetch-depth: 0 | ||
| - uses: ./fprime/.github/actions/setup | ||
| with: | ||
| location: ./fprime | ||
| - name: "Download and Setup AArch64 Linux Toolchain" | ||
| run: | | ||
| mkdir -p ${AARCH64_TOOLCHAIN_DIR} | ||
| wget -q ${AARCH64_TOOLCHAIN_URL} | ||
| tar -xf $(basename ${AARCH64_TOOLCHAIN_URL}) -C ${AARCH64_TOOLCHAIN_DIR} --strip-components 1 | ||
| echo "${AARCH64_TOOLCHAIN_DIR}/bin" >> $GITHUB_PATH | ||
| - name: "Generate AArch64 Linux Build Cache" | ||
| run: | | ||
| fprime-util generate aarch64-linux | ||
| - name: "Build AArch64 Linux" | ||
| run: | | ||
| fprime-util build aarch64-linux | ||
| - name: "Prepare artifacts" | ||
| run: | | ||
| mkdir -p aarch64-linux-artifacts-soak | ||
| cp -r ./build-artifacts aarch64-linux-artifacts-soak | ||
|
There was a problem hiding this comment. Could make |
||
| cp -r Components/Led/test/int aarch64-linux-artifacts-soak | ||
| # Copy monitoring script from F' repo workflows | ||
| if [ -f "./fprime/.github/workflows/soak_monitor.py" ]; then | ||
| cp ./fprime/.github/workflows/soak_monitor.py aarch64-linux-artifacts-soak/soak_monitor.py | ||
| fi | ||
| - name: 'Archive Build Artifacts' | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: aarch64-linux-artifacts-soak | ||
| path: aarch64-linux-artifacts-soak | ||
| retention-days: 5 | ||
|
|
||
| deploy-service: | ||
| name: "Deploy as System Service" | ||
| runs-on: [self-hosted, ARM64] | ||
| needs: cross-compilation | ||
| steps: | ||
| - name: "Stop existing services if running" | ||
| run: | | ||
| echo "Stopping existing services if they exist..." | ||
| # Stop and disable services, || true to avoid errors | ||
|
|
||
| sudo systemctl stop ${FSW_SERVICE_NAME} || true | ||
| sudo systemctl disable ${FSW_SERVICE_NAME} || true | ||
| sudo systemctl stop ${GDS_SERVICE_NAME} || true | ||
| sudo systemctl disable ${GDS_SERVICE_NAME} || true | ||
|
|
||
| - name: "Checkout F´ Repository" | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| sparse-checkout: 'requirements.txt' | ||
| sparse-checkout-cone-mode: false | ||
|
|
||
| - name: "Setup F´ Python Environment" | ||
| run: | | ||
| echo "Setting up F´ Python virtual environment..." | ||
| sudo mkdir -p ${INSTALL_DIR}/venv | ||
| sudo python3 -m venv ${INSTALL_DIR}/venv | ||
|
|
||
| # Activate virtual environment and install F´ requirements | ||
| sudo ${INSTALL_DIR}/venv/bin/pip install --upgrade pip | ||
| sudo ${INSTALL_DIR}/venv/bin/pip install -r requirements.txt | ||
|
|
||
| # Install additional packages needed for soak testing | ||
| sudo ${INSTALL_DIR}/venv/bin/pip install psutil | ||
|
|
||
| - name: "Artifacts Download" | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: aarch64-linux-artifacts-soak | ||
|
|
||
| - name: "Install new binary" | ||
| run: | | ||
| echo "Installing new binary to ${INSTALL_DIR}..." | ||
| sudo mkdir -p ${INSTALL_DIR}/bin | ||
| sudo mkdir -p ${INSTALL_DIR}/dict | ||
| sudo mkdir -p ${INSTALL_DIR}/test | ||
| sudo mkdir -p ${INSTALL_DIR}/logs | ||
| sudo mkdir -p ${INSTALL_DIR}/ComLoggerFiles | ||
| sudo mkdir -p ${INSTALL_DIR}/monitoring | ||
|
|
||
| # Copy binary, dict, and test files | ||
| sudo cp ./build-artifacts/aarch64-linux/LedBlinker/bin/LedBlinker ${INSTALL_DIR}/bin/ | ||
| sudo cp -r ./build-artifacts/aarch64-linux/LedBlinker/dict/* ${INSTALL_DIR}/dict/ | ||
| sudo cp -r ./int/* ${INSTALL_DIR}/test/ | ||
|
|
||
| # Copy monitoring script if it exists | ||
| if [ -f "./soak_monitor.py" ]; then | ||
| sudo cp ./soak_monitor.py ${INSTALL_DIR}/monitoring/ | ||
| sudo chmod +x ${INSTALL_DIR}/monitoring/soak_monitor.py | ||
| fi | ||
|
|
||
| # Clean ComLogger files directory to start fresh monitoring | ||
| echo "Cleaning ComLogger files directory for fresh monitoring session..." | ||
| sudo rm -rf ${INSTALL_DIR}/ComLoggerFiles/* | ||
|
|
||
| # Make binary executable | ||
| sudo chmod +x ${INSTALL_DIR}/bin/LedBlinker | ||
|
|
||
| # Set ownership, so service can access files | ||
| sudo chown -R ${SERVICE_USER}:${SERVICE_USER} ${INSTALL_DIR} | ||
|
|
||
| - name: "Create systemd service file" | ||
| run: | | ||
| echo "Creating systemd service file for LedBlinker..." | ||
| sudo tee /etc/systemd/system/${FSW_SERVICE_NAME}.service > /dev/null << EOF | ||
| [Unit] | ||
| Description=F' LED Blinker Application, system service meant for soak testing | ||
| After=network.target | ||
|
|
||
| [Service] | ||
| Type=simple | ||
| User=${SERVICE_USER} | ||
| WorkingDirectory=${INSTALL_DIR} | ||
| ExecStart=${INSTALL_DIR}/bin/LedBlinker -a 0.0.0.0 -p 50000 | ||
| StandardOutput=journal | ||
| StandardError=journal | ||
|
|
||
| [Install] | ||
| WantedBy=multi-user.target | ||
| EOF | ||
|
|
||
| echo "Creating systemd service file for GDS..." | ||
| sudo tee /etc/systemd/system/${GDS_SERVICE_NAME}.service > /dev/null << EOF | ||
| [Unit] | ||
| Description=F' GDS for LED Blinker soak testing | ||
| After=network.target ${FSW_SERVICE_NAME}.service | ||
|
|
||
| [Service] | ||
| Type=simple | ||
| User=${SERVICE_USER} | ||
| WorkingDirectory=${INSTALL_DIR} | ||
| ExecStart=${INSTALL_DIR}/venv/bin/fprime-gds --ip-client --ip-address 127.0.0.1 --ip-port 50000 -d ${INSTALL_DIR} -n --logs ${INSTALL_DIR}/logs --framing-selection fprime | ||
| StandardOutput=journal | ||
| StandardError=journal | ||
|
|
||
| [Install] | ||
| WantedBy=multi-user.target | ||
| EOF | ||
|
|
||
| - name: "Start and enable services" | ||
| run: | | ||
| echo "Starting and enabling services..." | ||
| sudo systemctl daemon-reload | ||
|
|
||
| # start and enable gds service | ||
| sudo systemctl enable ${GDS_SERVICE_NAME} | ||
| sudo systemctl start ${GDS_SERVICE_NAME} | ||
|
|
||
| # start and enable binary | ||
| sudo systemctl start ${FSW_SERVICE_NAME} | ||
| sudo systemctl enable ${FSW_SERVICE_NAME} | ||
|
|
||
| echo "Waiting for services to stabilize..." | ||
| sleep 5 # Wait for services to stabilize | ||
|
|
||
| echo "Checking final service status..." | ||
| sudo systemctl status ${FSW_SERVICE_NAME} --no-pager -l -n 50 | ||
| sudo systemctl status ${GDS_SERVICE_NAME} --no-pager -l -n 150 | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| # Scheduled testing of LED blinker system service every 30 minutes | ||
| # Analyzes ComLogger logs and runs integration tests | ||
| # | ||
| # This workflow analyzes ComLogger logs: Binary .com files from ComLogger component | ||
| # and runs integration tests | ||
| # | ||
| # Note: The "Artifact not found: ledblinker-monitoring-state" error is expected on the first run | ||
| # and is handled with continue-on-error: true | ||
| # | ||
| name: "Soak Test AArch64 Linux: External Repo LedBlinker" | ||
|
|
||
| on: | ||
| #schedule: | ||
| # Run every 30 minutes | ||
| #- cron: '*/15 * * * *' | ||
| # Allow manual triggering for testing | ||
| workflow_dispatch: | ||
|
|
||
| env: | ||
| FSW_SERVICE_NAME: ledblinker-soak-fsw | ||
| GDS_SERVICE_NAME: ledblinker-soak-gds | ||
| INSTALL_DIR: /opt/ledblinker-soak | ||
|
|
||
| jobs: | ||
| soak-monitoring: | ||
| name: "Soak Log Analysis" | ||
| runs-on: [self-hosted, ARM64] | ||
| steps: | ||
|
|
||
| - name: "Run soak monitoring" | ||
| run: | | ||
| # Use the virtual environment created during setup | ||
| . ${INSTALL_DIR}/venv/bin/activate | ||
|
|
||
| TEST_TIMESTAMP=$(date +%Y%m%d_%H%M%S) | ||
|
|
||
| echo "=== Soak monitoring started at $(date) (${TEST_TIMESTAMP}) ===" | ||
|
|
||
| # Run the soak monitor | ||
| cd ${INSTALL_DIR}/monitoring | ||
| python3 soak_monitor.py \ | ||
| --dictionary ${INSTALL_DIR}/dict/LedBlinkerTopologyDictionary.json \ | ||
| --com-logs ${INSTALL_DIR}/ComLoggerFiles \ | ||
|
|
||
|
|
||
|
|
||
| integration-test: | ||
| name: "Integration Test" | ||
| runs-on: [self-hosted, ARM64] | ||
| needs: soak-monitoring | ||
| steps: | ||
|
|
||
| - name: "Check service status" | ||
| run: | | ||
| echo "=== CHECKING SERVICES FOR INTEGRATION TEST ===" | ||
| echo "Checking if ${FSW_SERVICE_NAME} service is running..." | ||
| if sudo systemctl is-active --quiet ${FSW_SERVICE_NAME}; then | ||
| echo "✅ Service ${FSW_SERVICE_NAME} is running" | ||
| else | ||
| echo "❌ Service ${FSW_SERVICE_NAME} is not running" | ||
| sudo systemctl status ${FSW_SERVICE_NAME} --no-pager -l | ||
| echo "Recent FSW service logs:" | ||
| sudo journalctl -u ${FSW_SERVICE_NAME} --no-pager -n 20 | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "Checking if ${GDS_SERVICE_NAME} service is running..." | ||
| if sudo systemctl is-active --quiet ${GDS_SERVICE_NAME}; then | ||
| echo "✅ Service ${GDS_SERVICE_NAME} is running" | ||
| else | ||
| echo "❌ Service ${GDS_SERVICE_NAME} is not running" | ||
| sudo systemctl status ${GDS_SERVICE_NAME} --no-pager -l | ||
| echo "Recent GDS service logs:" | ||
| sudo journalctl -u ${GDS_SERVICE_NAME} --no-pager -n 20 | ||
| exit 1 | ||
| fi | ||
|
|
||
| - name: "Run integration tests" | ||
| run: | | ||
| # Use the virtual environment created during setup | ||
| . ${INSTALL_DIR}/venv/bin/activate | ||
|
|
||
| TEST_TIMESTAMP=$(date +%Y%m%d_%H%M%S) | ||
|
|
||
| echo "=== Integration test started at $(date) (${TEST_TIMESTAMP}) ===" | ||
|
|
||
| # Run the integration test | ||
| echo "Running integration tests..." | ||
| pytest --dictionary ${INSTALL_DIR}/dict/LedBlinkerTopologyDictionary.json ${INSTALL_DIR}/test/led_integration_tests.py | ||
|
|
||
| echo "=== Integration test completed at $(date) (${TEST_TIMESTAMP}) ===" |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
May want to have a group per release number (again via
github.event)