CI/CD pipeline agnostic of Workflow Engine and integrated on github actions

- Implemented ci.sh for orchestrating the complete build pipeline.
- Created ci_00_config.sh for centralized configuration of build scripts.
- Created ci_build_firmware.sh for building firmware for specified PlatformIO environments.
- Created ci_prepare_artifacts.sh for preparing firmware artifacts for upload or deployment.
- Created ci_set_version.sh for updating version tags in firmware configuration files.
- Created ci_build.sh to orchestrate the complete build pipeline.
- Created ci_qa.sh for code linting and formatting checks using clang-format.
- Created ci_site.sh for building and deploying VuePress documentation with version management.
- Implemented checks for required tools and dependencies in the new scripts.
- Improved internal scripts for better error handling and logging.

UPDATE the web installer manifest generation and update documentation structure
- Enhanced ci_list-env.sh to list environments from a JSON file.
- Replaced  common_wu.py and gen_wu.py scripts with new npm scripts for site generation and previewing on docsgen/gen_wu.js
- Replaced  generate_board_docs.py with docsgen/generated_board_docs.js
- Added new npm scripts for integration of site generation on build phase.
- Created preview_site.js to serve locally generated site over HTTPS with improved error handling.
- Added new CI environments for CI builds in environments.json.
- Deleted lint.yml as part of workflow cleanup.
- Enhanced task-build.yml to include linting as a job and added support for specifying PlatformIO version.
- Improved task-docs.yml to handle versioning more effectively and added clean option.

Enhance documentation
- ADD CLEAR Mark of development version of site
- Updated README.md to include detailed workflow dependencies and relationships using mermaid diagrams.
- Improved development.md with a quick checklist for contributors and clarified the code style guide.
- Enhanced quick_start.md with tips for contributors and streamlined the workflow explanation.

LINT FIX
- Refined User_config.h for better formatting consistency.
- Adjusted blufi.cpp and gatewayBT.cpp for improved code readability and consistency in formatting.
- Updated gatewaySERIAL.cpp and mqttDiscovery.cpp to enhance logging error messages.
- Improved sensorDS1820.cpp for better logging of device information.

Author: Odyno

.github/workflows/README.md

@@ -1,30 +1,65 @@
 name: Build
+# See details in .github/workflows/README.md (workflow docs)
+run-name: "CI build for ${{ github.ref_name }}"
 
 on: [push, pull_request]
 
 jobs:
+  determine-build-scope:
+    runs-on: ubuntu-latest
+    outputs:
+      environment-set: ${{ steps.decide.outputs.environment-set }}
+    steps:
+      - name: Decide environment set
+        id: decide
+        run: |
+          # Full build for: development, master, edge, stable, release/*, hotfix/*, and all PRs
+          if [[ "${{ github.event_name }}" == "pull_request" ]] || \
+             [[ "${{ github.ref }}" == "refs/heads/development" ]] || \
+             [[ "${{ github.ref }}" == "refs/heads/master" ]] || \
+             [[ "${{ github.ref }}" == "refs/heads/edge" ]] || \
+             [[ "${{ github.ref }}" == "refs/heads/stable" ]] || \
+             [[ "${{ github.ref }}" == refs/heads/release/* ]] || \
+             [[ "${{ github.ref }}" == refs/heads/hotfix/* ]]; then
+            echo "environment-set=all" >> $GITHUB_OUTPUT
+          else
+            echo "environment-set=ci" >> $GITHUB_OUTPUT
+          fi
+
+
   build:
+    needs: determine-build-scope
     name: Build firmware
     uses: ./.github/workflows/task-build.yml
     with:
-      python-version: '3.13'
-      enable-dev-ota: false
+      enable-dev-ota: true
       artifact-retention-days: 7
-      prepare-for-deploy: false
+      environment-set: ${{ needs.determine-build-scope.outputs.environment-set }}
 
   documentation:
+    needs: build
     name: Build documentation
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+
+      - name: Download all firmware artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "*"
+          path: generated/artifacts
+          merge-multiple: true
+
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
           node-version: "14.x"
-      - name: Download Common Config
-        run: |
-          curl -o docs/.vuepress/public/commonConfig.js https://www.theengs.io/commonConfig.js
-      - name: Install build dependencies
+
+      - name: Install docs dependencies
         run: npm install
-      - name: Build documentation
-        run: npm run docs:build
+
+      - name: Build documentation with ci_site.sh 
+
+        run: |
+          # Quick build check (dev mode) not used task-docs.yml because is not required the upload of artefacts
+          ./scripts/ci.sh site --mode dev 

.github/workflows/build.yml

@@ -3,7 +3,7 @@ name: Build binaries, docs and publish to dev folder
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 0 * * *'
+    - cron: "0 0 * * *"
 
 jobs:
   prepare:
@@ -18,86 +18,33 @@ jobs:
         with:
           length: 6
 
-  build:
+  handle-firmwares:
     needs: prepare
-    name: Build development firmware
+    name: Build and deploy development firmwares artefacts
     uses: ./.github/workflows/task-build.yml
     with:
-      python-version: '3.13'
       enable-dev-ota: true
       version-tag: ${{ needs.prepare.outputs.short-sha }}
       artifact-retention-days: 1
-      artifact-name-prefix: 'firmware-'
-      prepare-for-deploy: true
 
-  deploy:
-    needs: [prepare, build]
+  handle-documentation:
+    needs: [prepare, handle-firmwares]
     runs-on: ubuntu-latest
-    if: github.repository_owner == '1technophile'
-    name: Deploy binaries and docs
     steps:
-      - uses: actions/checkout@v4
-
       - name: Download all firmware artifacts
         uses: actions/download-artifact@v4
         with:
-          pattern: firmware-*
-          path: toDeploy
+          pattern: "*"
+          path: generated/artifacts
           merge-multiple: true
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
+      - name: Build and deploy development documentation
+        uses: ./.github/workflows/task-docs.yml
         with:
-          version: "latest"
-          enable-cache: false
-
-      - name: Install dependencies
-        run: |
-          uv pip install --system -U https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip
-
-      - name: Create library zips
-        run: |
-          # Install libraries for a representative environment to get libdeps
-          platformio pkg install -e esp32dev-ble --no-save
-          cd .pio/libdeps
-          # Replace spaces with underscores in folder names
-          find . -type d -name "* *" | while read FNAME; do mv "$FNAME" "${FNAME// /_}"; done
-          # Zip libraries per board
-          for i in */; do
-            zip -r "${i%/}-libraries.zip" "$i"
-          done
-          mv *.zip ../../toDeploy/
-
-      - name: Prepare additional assets
-        run: |
-          cd toDeploy
-          # Remove binaries for *-all*, *-test* env
-          rm -f *-all*.bin *-test*.bin *-test*.zip || true
-          cd ..
-          # Zip source code
-          zip -r toDeploy/OpenMQTTGateway_sources.zip main LICENSE.txt
-          ls -lA toDeploy/
-
-  documentation:
-    needs: [prepare, build]
-    name: Build and deploy development documentation
-    uses: ./.github/workflows/task-docs.yml
-    with:
-      python-version: '3.11'
-      node-version: '16.x'
-      version-source: 'custom'
-      custom-version: 'DEVELOPMENT SHA:${{ needs.prepare.outputs.short-sha }} TEST ONLY'
-      base-path: '/dev/'
-      destination-dir: 'dev'
-      generate-webuploader: true
-      webuploader-args: '--dev'
-      run-pagespeed: true
-      pagespeed-url: 'https://docs.openmqttgateway.com/dev/'
-    secrets:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      APIKEY: ${{ secrets.APIKEY }}
+          mode: "dev"
+          version: ${{ needs.prepare.outputs.short-sha }}
+          url-prefix: "/dev/"
+          destination-dir: "dev"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-pagespeed: true
+          pagespeed-url: 'https://docs.openmqttgateway.com/dev/'
+          pagespeed-url-apikey: ${{ secrets.APIKEY }}

.github/workflows/build_and_docs_to_dev.yml

@@ -88,19 +88,22 @@
             "sonoff-basic",
             "sonoff-basic-rfr3"
         ],
-        "metadata": {
-            "totalCount": 83,
-            "categories": {
-                "esp32": 42,
-                "esp8266": 17,
-                "specialized": 24
-            },
-            "notes": [
-                "This unified list contains all environments from both build.yml and build_and_docs_to_dev.yml",
-                "Previously build.yml had 85 environments and build_and_docs_to_dev.yml had 88",
-                "After deduplication and proper ordering, the total is 83 unique environments",
-                "Environments ending in -test or -all-test are typically excluded from production releases"
-            ]
-        }
+        "ci": [
+            "esp32dev-all-test",
+            "esp32dev-ble",
+            "esp32dev-ir",
+            "esp32dev-rf",
+            "esp32dev-rtl_433",
+            "esp32s3-dev-c1-ble",
+            "esp32c3-dev-m1-ble",
+            "heltec-ble",
+            "lilygo-ble",
+            "nodemcuv2-all-test",
+            "nodemcuv2-ir",
+            "nodemcuv2-rf",
+            "theengs-bridge",
+            "theengs-plug",
+            "rfbridge"
+        ]
     }
 }

.github/workflows/environments.json

@@ -9,13 +9,9 @@ jobs:
     name: Build and deploy production documentation
     uses: ./.github/workflows/task-docs.yml
     with:
-      python-version: '3.11'
-      node-version: '14.x'
-      version-source: 'release'
-      base-path: '/'
-      destination-dir: '.'
-      generate-webuploader: true
-      webuploader-args: '${GITHUB_REPOSITORY}'
-      run-pagespeed: false
+      mode: "prod"
+      version: "auto" # Will detect latest git tag
+      url-prefix: "/"
+      destination-dir: "."
     secrets:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
           VERSION_TAG=${GITHUB_REF#refs/tags/}
           echo "version=${VERSION_TAG}" >> $GITHUB_OUTPUT
           echo "Extracted version: ${VERSION_TAG}"
-      
+
       - name: Extract release info
         id: extract-release
         run: |
@@ -33,79 +33,43 @@ jobs:
     name: Build release firmware
     uses: ./.github/workflows/task-build.yml
     with:
-      python-version: '3.13'
       enable-dev-ota: false
       version-tag: ${{ needs.prepare.outputs.version-tag }}
       artifact-retention-days: 90
-      artifact-name-prefix: 'firmware-'
-      prepare-for-deploy: true
 
   deploy:
     needs: [prepare, build]
     runs-on: ubuntu-latest
     name: Deploy release assets
     steps:
-      - uses: actions/checkout@v4
-
       - name: Download all firmware artifacts
         uses: actions/download-artifact@v4
         with:
-          pattern: firmware-*
-          path: .pio/build
-          merge-multiple: false
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.13"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-        with:
-          version: "latest"
-          enable-cache: false
-
-      - name: Install platformio
-        run: |
-          uv pip install --system -U https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip
-
-      - name: Reorganize artifacts for prepare_deploy.sh
-        run: |
-          # Move artifacts from download structure to expected .pio/build structure
-          for env_dir in .pio/build/firmware-*; do
-            if [ -d "$env_dir" ]; then
-              env_name=$(basename "$env_dir" | sed 's/^firmware-//')
-              mkdir -p ".pio/build/${env_name}"
-              mv "$env_dir"/*.bin ".pio/build/${env_name}/" 2>/dev/null || true
-              rm -rf "$env_dir"
-            fi
-          done
-
-      - name: Prepare Release Assets
-        run: |
-          sudo apt install rename
-          ./scripts/prepare_deploy.sh
+          pattern: "*"
+          path: generated/artifacts
+          merge-multiple: true
 
       - name: Upload Release Assets
         uses: bgpat/release-asset-action@03b0c30db1c4031ce3474740b0e4275cd7e126a3
         with:
-          pattern: "toDeploy/*"
+          pattern: "generated/artifacts/*"
           github-token: ${{ secrets.GITHUB_TOKEN }}
           release-url: ${{ needs.prepare.outputs.upload-url }}
           allow-overwrite: true
 
   documentation:
     needs: [prepare, deploy]
-    name: Build and deploy release documentation
-    uses: ./.github/workflows/task-docs.yml
-    with:
-      python-version: '3.11'
-      node-version: '18.x'
-      version-source: 'git-tag'
-      base-path: '/'
-      destination-dir: '.'
-      generate-webuploader: true
-      run-pagespeed: false
-    secrets:
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      APIKEY: ${{ secrets.APIKEY }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all firmware artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: "*"
+      - name: Build and deploy release documentation
+        uses: ./.github/workflows/task-docs.yml
+        with:
+          mode: "prod"
+          version: ${{ needs.prepare.outputs.version-tag }}
+          url-prefix: "/"
+          destination-dir: "."
+          github-token: ${{ secrets.GITHUB_TOKEN }}