diff --git a/.github/actions/common/code-style/action.yml b/.github/actions/common/code-style/action.yml index a3cf43a44c..89f39e924c 100644 --- a/.github/actions/common/code-style/action.yml +++ b/.github/actions/common/code-style/action.yml @@ -5,6 +5,14 @@ inputs: description: "Directory to check for C/C++ files" required: false default: "." + name: + description: 'Name for the output artifact' + required: false + default: 'code-style-check-report' + fail-on-findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" runs: using: "composite" @@ -19,16 +27,51 @@ runs: shell: bash - name: Run code style check + id: code-style-check env: target_dir: ${{ inputs.target_dir }} run: | - chmod +x .github/actions/common/code-style/entrypoint.sh - ./.github/actions/common/code-style/entrypoint.sh "${target_dir}" + chmod +x .github/actions/common/code-style/entrypoint.sh + ./.github/actions/common/code-style/entrypoint.sh "${target_dir}" || echo "STYLE_ISSUES=true" >> $GITHUB_OUTPUT shell: bash - - name: Upload clang-format report on failure - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #4.6.2 + - name: Analyze code style results + if: always() + run: | + if [ "${{ steps.code-style-check.outputs.STYLE_ISSUES }}" == "true" ]; then + # Count number of files with style issues + if [ -f "_output/diff.html" ]; then + # Try to count files from diff output + file_count=$(diff -u --recursive "${{ inputs.target_dir }}" "_styled/${{ inputs.target_dir }}" 2>/dev/null | grep -c "^diff -u" || echo "1+") + + echo "### Code Style Check Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- ❌ **Status**: Style issues found" >> $GITHUB_STEP_SUMMARY + echo "- πŸ“ **Files affected**: ${file_count}" >> $GITHUB_STEP_SUMMARY + echo "- πŸ“„ **Detailed report**: Available in artifacts (diff.html)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **Please review the code-style report artifact and apply clang-format to fix the issues.**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "πŸ’‘ **Tip**: Run \`clang-format -i\` on the affected files to automatically fix formatting." >> $GITHUB_STEP_SUMMARY + fi + else + echo "### Code Style Check Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "βœ… **All code follows the style guidelines!**" >> $GITHUB_STEP_SUMMARY + fi + shell: bash + + - name: Upload clang-format report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: code-style-report + name: ${{ inputs.name }} path: _output/diff.html + if-no-files-found: ignore + + - name: Fail if code style issues found + if: inputs.fail-on-findings == 'true' && steps.code-style-check.outputs.STYLE_ISSUES == 'true' + shell: bash + run: | + echo "❌ Code style issues found. Failing the job." + exit 1 diff --git a/.github/actions/common/hadolint/action.yaml b/.github/actions/common/hadolint/action.yaml index 525a43ccdf..c3d3fa4176 100644 --- a/.github/actions/common/hadolint/action.yaml +++ b/.github/actions/common/hadolint/action.yaml @@ -18,6 +18,10 @@ inputs: github_token: description: GitHub token for ReviewDog required: false + fail-on-findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" runs: using: "composite" @@ -30,15 +34,70 @@ runs: shell: bash - name: Run Hadolint - env: + id: run-hadolint + env: dockerfile: ${{ inputs.dockerfile }} - output-file: ${{ inputs.output-file }} + output_file: ${{ inputs.output-file }} run: | hadolint ${dockerfile} \ --format tty \ - | tee ${output-file} + 2>&1 | tee ${output_file} || true + if [ ! -f "${output_file}" ]; then + echo "No Dockerfile found or hadolint produced no output" > ${output_file} + fi + shell: bash + + - name: Analyze Hadolint results + if: always() + env: + output_file: ${{ inputs.output-file }} + name: ${{ inputs.name }} + run: | + if [ -f "${output_file}" ]; then + # Count issues by severity (hadolint format: DL#### or SC#### followed by colored severity) + # Pattern matches: DL3008 or SC1091 (hadolint and shellcheck codes) + error_count=$(grep -E "(DL|SC)[0-9]+" "${output_file}" 2>/dev/null | grep -i "error" | wc -l | tr -d '[:space:]' || echo "0") + warning_count=$(grep -E "(DL|SC)[0-9]+" "${output_file}" 2>/dev/null | grep -i "warning" | wc -l | tr -d '[:space:]' || echo "0") + info_count=$(grep -E "(DL|SC)[0-9]+" "${output_file}" 2>/dev/null | grep -i "info" | wc -l | tr -d '[:space:]' || echo "0") + style_count=$(grep -E "(DL|SC)[0-9]+" "${output_file}" 2>/dev/null | grep -i "style" | wc -l | tr -d '[:space:]' || echo "0") + # Ensure counts are valid integers, default to 0 if empty + error_count=${error_count:-0} + warning_count=${warning_count:-0} + info_count=${info_count:-0} + style_count=${style_count:-0} + # Additional safety check - ensure numeric (use case to validate) + case "$error_count" in ''|*[!0-9]*) error_count=0 ;; esac || true + case "$warning_count" in ''|*[!0-9]*) warning_count=0 ;; esac || true + case "$info_count" in ''|*[!0-9]*) info_count=0 ;; esac || true + case "$style_count" in ''|*[!0-9]*) style_count=0 ;; esac || true + total=$((error_count + warning_count + info_count + style_count)) || total=0 + + echo "### Hadolint Results for ${name}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Total Issues**: $total" >> $GITHUB_STEP_SUMMARY + + if [ "$error_count" -gt 0 ]; then + echo "- ❌ **Errors**: $error_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$warning_count" -gt 0 ]; then + echo "- ⚠️ **Warnings**: $warning_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$info_count" -gt 0 ]; then + echo "- ℹ️ **Info**: $info_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$style_count" -gt 0 ]; then + echo "- 🎨 **Style**: $style_count" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$total" -gt 0 ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **Please review the Hadolint report artifact and consider fixing the issues.**" >> $GITHUB_STEP_SUMMARY + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "βœ… **No issues found!**" >> $GITHUB_STEP_SUMMARY + fi + fi shell: bash - continue-on-error: true - name: Upload Hadolint report as artifact if: always() @@ -55,3 +114,17 @@ runs: reporter: github-pr-review level: warning hadolint_flags: ${{ inputs.dockerfile }} + + - name: Fail if Hadolint found issues + if: inputs.fail-on-findings == 'true' + shell: bash + env: + output_file: ${{ inputs.output-file }} + run: | + if [ -f "${output_file}" ]; then + issue_count=$(grep -E "(DL|SC)[0-9]+" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + if [ "$issue_count" -gt 0 ]; then + echo "❌ Hadolint found $issue_count issue(s). Failing the job." + exit 1 + fi + fi diff --git a/.github/actions/common/license-namespace-checker/action.yaml b/.github/actions/common/license-namespace-checker/action.yaml index 1b7d7ca135..b49505d42e 100644 --- a/.github/actions/common/license-namespace-checker/action.yaml +++ b/.github/actions/common/license-namespace-checker/action.yaml @@ -1,20 +1,28 @@ name: 'License and Namespace Checker' description: 'Checks license headers and namespace usage in headers' +inputs: + name: + description: 'Name for the output artifact' + required: false + default: 'license-namespace-check-report' + path: + description: 'Path to the repository root' + required: false + default: '.' + fail-on-findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" runs: using: 'composite' steps: - - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 - with: - path: edge-ai-libraries-repo - persist-credentials: false - fetch-depth: 0 - - name: Get list of changed files shell: bash id: discover-changes + env: + REPO_PATH: ${{ inputs.path }} run: | - cd edge-ai-libraries-repo + cd "${REPO_PATH}" if [ "$(git rev-parse --abbrev-ref HEAD)" != "main" ]; then git fetch origin main:main echo "Fetched main branch" @@ -33,12 +41,68 @@ runs: fi - name: Check License header and namespace usage in headers + id: license-check shell: bash env: CHANGED_FILES: ${{ steps.discover-changes.outputs.changed_files }} + REPO_PATH: ${{ inputs.path }} + output_file: license-check-report.txt run: | if [ -z "${CHANGED_FILES}" ]; then - echo "No new files to scan." + echo "No new files to scan." | tee "${output_file}" + echo "ISSUES_FOUND=false" >> $GITHUB_OUTPUT else - ./.github/actions/common/license-namespace-checker/run.sh . $CHANGED_FILES + if "${GITHUB_ACTION_PATH}/run.sh" "${REPO_PATH}" $CHANGED_FILES 2>&1 | tee "${output_file}"; then + echo "ISSUES_FOUND=false" >> $GITHUB_OUTPUT + else + echo "ISSUES_FOUND=true" >> $GITHUB_OUTPUT + fi fi + + - name: Upload License Check report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.name }} + path: license-check-report.txt + if-no-files-found: warn + + - name: Analyze License Check results + if: always() + shell: bash + env: + output_file: license-check-report.txt + run: | + if [ "${{ steps.license-check.outputs.ISSUES_FOUND }}" == "true" ]; then + # Count files with issues + if [ -f "${output_file}" ]; then + error_count=$(grep -c "Error:" "${output_file}" 2>/dev/null || echo "0") + echo "### License & Namespace Check Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- ❌ **Status**: Issues found" >> $GITHUB_STEP_SUMMARY + echo "- πŸ” **Total errors**: ${error_count}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "πŸ“„ **See job logs for detailed error messages.**" >> $GITHUB_STEP_SUMMARY + fi + elif [ "${{ steps.discover-changes.outputs.changed_files }}" != "" ]; then + echo "### License & Namespace Check Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "βœ… **All checked files have correct license headers and namespace usage!**" >> $GITHUB_STEP_SUMMARY + else + echo "### License & Namespace Check Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "ℹ️ **No relevant files changed - check skipped**" >> $GITHUB_STEP_SUMMARY + fi + + - name: Fail if license/namespace issues found + if: inputs.fail-on-findings == 'true' && steps.license-check.outputs.ISSUES_FOUND == 'true' + shell: bash + run: | + echo "❌ License or namespace issues found. Failing the job." + exit 1 + + - name: Clean up + if: always() + shell: bash + run: | + rm -f license-check-report.txt diff --git a/.github/actions/common/pylint/action.yaml b/.github/actions/common/pylint/action.yaml index 80d73917c4..361e111618 100644 --- a/.github/actions/common/pylint/action.yaml +++ b/.github/actions/common/pylint/action.yaml @@ -18,6 +18,10 @@ inputs: github_token: description: GitHub token for ReviewDog required: false + fail-on-findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" runs: using: "composite" @@ -34,14 +38,90 @@ runs: shell: bash - name: Run pylint + id: run-pylint env: path: ${{ inputs.path }} - output-file: ${{ inputs.output-file }} + output_file: ${{ inputs.output-file }} + run: | + # Run pylint on all Python files at once for a single comprehensive score + echo "πŸ” Searching for Python files in: ${path}" + python_files=$(find "${path}" -name "*.py" -not -path "*/venv/*" 2>/dev/null || true) + + if [ -n "${python_files}" ]; then + echo "πŸ“ Found Python files, running pylint..." + find "${path}" -name "*.py" -not -path "*/venv/*" -print0 | xargs -0 pylint 2>&1 | tee "${output_file}" || true + else + echo "⚠️ No Python files found in ${path}" | tee "${output_file}" + fi + + if [ ! -f "${output_file}" ]; then + echo "No Python files found or pylint produced no output" > "${output_file}" + fi + shell: bash + + - name: Analyze Pylint results + if: always() + env: + output_file: ${{ inputs.output-file }} + name: ${{ inputs.name }} run: | - find ${path} -name "*.py" -not -path "*/venv/*" \ - | xargs pylint | tee ${output-file} + if [ -f "${output_file}" ]; then + # Count issues by severity (pylint uses C/R/W/E/F prefixes) - ensure we get single clean numbers + convention_count=$(grep -E "^[^:]+:[0-9]+:[0-9]+: C[0-9]+:" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + refactor_count=$(grep -E "^[^:]+:[0-9]+:[0-9]+: R[0-9]+:" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + warning_count=$(grep -E "^[^:]+:[0-9]+:[0-9]+: W[0-9]+:" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + error_count=$(grep -E "^[^:]+:[0-9]+:[0-9]+: E[0-9]+:" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + fatal_count=$(grep -E "^[^:]+:[0-9]+:[0-9]+: F[0-9]+:" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + # Ensure counts are valid integers, default to 0 if empty + convention_count=${convention_count:-0} + refactor_count=${refactor_count:-0} + warning_count=${warning_count:-0} + error_count=${error_count:-0} + fatal_count=${fatal_count:-0} + # Additional safety check - ensure numeric (use case to validate) - set returns true + case "$convention_count" in ''|*[!0-9]*) convention_count=0 ;; esac || true + case "$refactor_count" in ''|*[!0-9]*) refactor_count=0 ;; esac || true + case "$warning_count" in ''|*[!0-9]*) warning_count=0 ;; esac || true + case "$error_count" in ''|*[!0-9]*) error_count=0 ;; esac || true + case "$fatal_count" in ''|*[!0-9]*) fatal_count=0 ;; esac || true + total=$((convention_count + refactor_count + warning_count + error_count + fatal_count)) || total=0 + + # Try to extract the score - ensure this doesn't fail + score=$(grep "Your code has been rated at" "${output_file}" 2>/dev/null | tail -1 | grep -oE "[0-9]+\.[0-9]+/10" 2>/dev/null || echo "") + + echo "### Pylint Results for ${name}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Total Issues**: $total" >> $GITHUB_STEP_SUMMARY + + if [ "$fatal_count" -gt 0 ]; then + echo "- πŸ”΄ **Fatal**: $fatal_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$error_count" -gt 0 ]; then + echo "- ❌ **Errors**: $error_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$warning_count" -gt 0 ]; then + echo "- ⚠️ **Warnings**: $warning_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$refactor_count" -gt 0 ]; then + echo "- πŸ”§ **Refactor**: $refactor_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$convention_count" -gt 0 ]; then + echo "- πŸ“‹ **Convention**: $convention_count" >> $GITHUB_STEP_SUMMARY + fi + + if [ -n "$score" ]; then + echo "- πŸ“Š **Score**: $score" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$total" -gt 0 ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **Please review the Pylint report artifact and consider fixing the issues.**" >> $GITHUB_STEP_SUMMARY + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "βœ… **No issues found!**" >> $GITHUB_STEP_SUMMARY + fi + fi shell: bash - continue-on-error: true - name: Upload Pylint report if: always() @@ -59,3 +139,16 @@ runs: level: warning workdir: ${{ inputs.path }} + - name: Fail if Pylint found issues + if: inputs.fail-on-findings == 'true' + shell: bash + env: + output_file: ${{ inputs.output-file }} + run: | + if [ -f "${output_file}" ]; then + issue_count=$(grep -E "^[^:]+:[0-9]+:[0-9]+: [CRWEF][0-9]+:" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + if [ "$issue_count" -gt 0 ]; then + echo "❌ Pylint found $issue_count issue(s). Failing the job." + exit 1 + fi + fi diff --git a/.github/actions/common/shellcheck/action.yaml b/.github/actions/common/shellcheck/action.yaml index 0258205c68..18c1527f0b 100644 --- a/.github/actions/common/shellcheck/action.yaml +++ b/.github/actions/common/shellcheck/action.yaml @@ -18,6 +18,10 @@ inputs: github_token: description: GitHub token for ReviewDog required: false + fail-on-findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" runs: using: "composite" @@ -27,13 +31,67 @@ runs: shell: bash - name: Run ShellCheck + id: run-shellcheck env: path: ${{ inputs.path }} - output-file: ${{ inputs.output-file }} + output_file: ${{ inputs.output-file }} + run: | + find ${path} -name "*.sh" -exec shellcheck {} \; 2>&1 | tee ${output_file} || true + if [ ! -f "${output_file}" ]; then + echo "No shell scripts found or shellcheck produced no output" > ${output_file} + fi + shell: bash + + - name: Analyze ShellCheck results + if: always() + env: + output_file: ${{ inputs.output-file }} + name: ${{ inputs.name }} run: | - find ${path} -name "*.sh" | xargs shellcheck | tee ${output-file} + if [ -f "${output_file}" ]; then + # Count issues by severity - shellcheck format is "SC#### (level):" + error_count=$(grep -E "SC[0-9]+ \(error\):" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + warning_count=$(grep -E "SC[0-9]+ \(warning\):" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + info_count=$(grep -E "SC[0-9]+ \(info\):" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + note_count=$(grep -E "SC[0-9]+ \(note\):" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + # Ensure counts are valid integers, default to 0 if empty + error_count=${error_count:-0} + warning_count=${warning_count:-0} + info_count=${info_count:-0} + note_count=${note_count:-0} + # Additional safety check - ensure numeric (use case to validate) + case "$error_count" in ''|*[!0-9]*) error_count=0 ;; esac || true + case "$warning_count" in ''|*[!0-9]*) warning_count=0 ;; esac || true + case "$info_count" in ''|*[!0-9]*) info_count=0 ;; esac || true + case "$note_count" in ''|*[!0-9]*) note_count=0 ;; esac || true + total=$((error_count + warning_count + info_count + note_count)) || total=0 + + echo "### ShellCheck Results for ${name}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Total Issues**: $total" >> $GITHUB_STEP_SUMMARY + + if [ "$error_count" -gt 0 ]; then + echo "- πŸ”΄ **Errors**: $error_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$warning_count" -gt 0 ]; then + echo "- ⚠️ **Warnings**: $warning_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$info_count" -gt 0 ]; then + echo "- ℹ️ **Info**: $info_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$note_count" -gt 0 ]; then + echo "- πŸ“ **Notes**: $note_count" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$total" -gt 0 ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **Please review the ShellCheck report artifact and consider fixing the issues.**" >> $GITHUB_STEP_SUMMARY + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "βœ… **No issues found!**" >> $GITHUB_STEP_SUMMARY + fi + fi shell: bash - continue-on-error: true - name: Upload ShellCheck report if: always() @@ -44,9 +102,22 @@ runs: - name: Run ReviewDog (ShellCheck) if: ${{ inputs.enable-reviewdog == 'true' }} - uses: reviewdog/action-shellcheck@5ebd09ddbe2ebb471646ce234c6c8dd18663ca7c # 1.30.0 + uses: reviewdog/action-shellcheck@4c07458293ac342d477251099501a718ae5ef86e # 1.32.0 with: github_token: ${{ inputs.github_token }} reporter: github-pr-review level: warning + - name: Fail if ShellCheck found issues + if: inputs.fail-on-findings == 'true' + shell: bash + env: + output_file: ${{ inputs.output-file }} + run: | + if [ -f "${output_file}" ]; then + issue_count=$(grep -E "SC[0-9]+" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + if [ "$issue_count" -gt 0 ]; then + echo "❌ ShellCheck found $issue_count issue(s). Failing the job." + exit 1 + fi + fi diff --git a/.github/actions/common/trivy-image-scan/action.yaml b/.github/actions/common/trivy-image-scan/action.yaml index a243f8e63f..3fb812a6be 100644 --- a/.github/actions/common/trivy-image-scan/action.yaml +++ b/.github/actions/common/trivy-image-scan/action.yaml @@ -1,6 +1,5 @@ name: "Trivy Image Scan" description: "Scan a Docker image using Trivy with configurable severity levels and output options" -#This action will fail if any vulnerbailities will be found inputs: image: description: "Docker image to scan (e.g. nginx:latest)" @@ -20,13 +19,19 @@ inputs: description: "File path to save Trivy report (optional)" required: false type: string + fail-on-findings: + description: "Whether to fail the action if vulnerabilities are found" + required: false + default: "true" + type: string runs: using: "docker" image: "Dockerfile" - entrypoint: "/entrypoint.sh" + entrypoint: "./entrypoint.sh" args: - ${{ inputs.image }} - ${{ inputs.severity }} - ${{ inputs.output-format }} - ${{ inputs.output-file }} + - ${{ inputs.fail-on-findings }} diff --git a/.github/actions/common/trivy-image-scan/entrypoint.sh b/.github/actions/common/trivy-image-scan/entrypoint.sh index 363aec5341..47461b6854 100644 --- a/.github/actions/common/trivy-image-scan/entrypoint.sh +++ b/.github/actions/common/trivy-image-scan/entrypoint.sh @@ -10,14 +10,23 @@ IMAGE="$1" SEVERITY="${2:-LOW,MEDIUM,HIGH,CRITICAL}" OUTPUT_FORMAT="${3:-table}" OUTPUT_FILE="$4" +FAIL_ON_FINDINGS="${5:-true}" echo "πŸ” Scanning image: $IMAGE" echo "⚠️ Severity filter: $SEVERITY" echo "πŸ“„ Output format: $OUTPUT_FORMAT" +echo "🚨 Fail on findings: $FAIL_ON_FINDINGS" + +# Determine exit code parameter based on fail-on-findings +if [ "$FAIL_ON_FINDINGS" = "true" ]; then + EXIT_CODE_PARAM="--exit-code 1" +else + EXIT_CODE_PARAM="--exit-code 0" +fi if [ -n "$OUTPUT_FILE" ]; then echo "πŸ’Ύ Saving report to: $OUTPUT_FILE" - trivy image --severity "$SEVERITY" --format "$OUTPUT_FORMAT" --output "$OUTPUT_FILE" --exit-code 1 "$IMAGE" + trivy image --severity "$SEVERITY" --format "$OUTPUT_FORMAT" --output "$OUTPUT_FILE" $EXIT_CODE_PARAM "$IMAGE" else - trivy image --severity "$SEVERITY" --format "$OUTPUT_FORMAT" --exit-code 1 "$IMAGE" + trivy image --severity "$SEVERITY" --format "$OUTPUT_FORMAT" $EXIT_CODE_PARAM "$IMAGE" fi diff --git a/.github/actions/common/yamllint/action.yaml b/.github/actions/common/yamllint/action.yaml index 3a717e538b..667319fac3 100644 --- a/.github/actions/common/yamllint/action.yaml +++ b/.github/actions/common/yamllint/action.yaml @@ -18,6 +18,10 @@ inputs: github_token: description: GitHub token for ReviewDog required: false + fail-on-findings: + description: "Whether to fail the action if issues are found" + required: false + default: "true" runs: using: "composite" @@ -27,13 +31,57 @@ runs: shell: bash - name: Run Yamllint + id: run-yamllint env: path: ${{ inputs.path }} - output-file: ${{ inputs.output-file }} + output_file: ${{ inputs.output-file }} run: | - yamllint ${path} | tee ${output-file} + yamllint ${path} 2>&1 | tee ${output_file} || true + if [ ! -f "${output_file}" ]; then + echo "No YAML files found or yamllint produced no output" > ${output_file} + fi + echo "βœ… Report created at: $(pwd)/${output_file}" + ls -lh ${output_file} + shell: bash + + - name: Analyze Yamllint results + if: always() + env: + output_file: ${{ inputs.output-file }} + name: ${{ inputs.name }} + run: | + if [ -f "${output_file}" ]; then + # Count issues by severity (yamllint uses ::error and ::warning in GitHub Actions format) + error_count=$(grep "::error" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + warning_count=$(grep "::warning" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + # Ensure counts are valid integers, default to 0 if empty + error_count=${error_count:-0} + warning_count=${warning_count:-0} + # Additional safety check - ensure numeric (use case to validate) + case "$error_count" in ''|*[!0-9]*) error_count=0 ;; esac || true + case "$warning_count" in ''|*[!0-9]*) warning_count=0 ;; esac || true + total=$((error_count + warning_count)) || total=0 + + echo "### Yamllint Results for ${name}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Total Issues**: $total" >> $GITHUB_STEP_SUMMARY + + if [ "$error_count" -gt 0 ]; then + echo "- ❌ **Errors**: $error_count" >> $GITHUB_STEP_SUMMARY + fi + if [ "$warning_count" -gt 0 ]; then + echo "- ⚠️ **Warnings**: $warning_count" >> $GITHUB_STEP_SUMMARY + fi + + if [ "$total" -gt 0 ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **Please review the Yamllint report artifact and consider fixing the issues.**" >> $GITHUB_STEP_SUMMARY + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "βœ… **No issues found!**" >> $GITHUB_STEP_SUMMARY + fi + fi shell: bash - continue-on-error: true - name: Upload Yamllint report if: always() @@ -49,3 +97,17 @@ runs: github_token: ${{ inputs.github_token }} reporter: github-pr-review level: warning + + - name: Fail if Yamllint found issues + if: inputs.fail-on-findings == 'true' + shell: bash + env: + output_file: ${{ inputs.output-file }} + run: | + if [ -f "${output_file}" ]; then + issue_count=$(grep -E "::error|::warning" "${output_file}" 2>/dev/null | wc -l | tr -d '[:space:]' || echo "0") + if [ "$issue_count" -gt 0 ]; then + echo "❌ Yamllint found $issue_count issue(s). Failing the job." + exit 1 + fi + fi diff --git a/.github/workflows/dls-build-and-test-deb_pkgs-and-deb_imgs.yaml b/.github/workflows/dls-build-and-test-deb_pkgs-and-deb_imgs.yaml index 636ef62402..035f6157a2 100644 --- a/.github/workflows/dls-build-and-test-deb_pkgs-and-deb_imgs.yaml +++ b/.github/workflows/dls-build-and-test-deb_pkgs-and-deb_imgs.yaml @@ -47,7 +47,7 @@ jobs: path: dl-streamer-tests-repo - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo @@ -59,10 +59,10 @@ jobs: # ======================================================== BUILDING PART ======================================================== - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #3.11.1 - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -155,7 +155,7 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #4.6.2 if: steps.test_hello_dlstreamer_script.outcome == 'success' with: - name: hello_dlstreamer_output_video_${{ matrix.runner_print_label }}_${{ matrix.ubuntu_version }}_CPU_yolo11s + name: DLS_hello_dlstreamer_output_video_${{ matrix.runner_print_label }}_${{ matrix.ubuntu_version }}_CPU_yolo11s path: ${{ env.DLS_REL_PATH }}/test_hello_dlstreamer - name: Init tests configurations @@ -213,14 +213,14 @@ jobs: --video-examples-path=${{ env.VIDEO_INPUTS_PATH }} \ --test-configs="$TEST_CONFIGS" \ --results-path="$DLS_TESTS_RESULTS_PATH" \ - --report-name="functional_tests_${{ matrix.runner_print_label }}_on_host_${{ matrix.ubuntu_version }}_results" \ + --report-name="DLS_functional_tests_${{ matrix.runner_print_label }}_on_host_${{ matrix.ubuntu_version }}_results" \ --on-host - name: Upload test results uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #4.6.2 if: steps.run_tests.outcome == 'success' with: - name: functional_tests_${{ matrix.runner_print_label }}_on_host_${{ matrix.ubuntu_version }}_results + name: DLS_functional_tests_${{ matrix.runner_print_label }}_on_host_${{ matrix.ubuntu_version }}_results path: "${{ env.DLS_TESTS_RESULTS_PATH }}" - name: Print tests summary in workflow @@ -229,7 +229,7 @@ jobs: dls_tests_results_path: ${{ env.DLS_TESTS_RESULTS_PATH }} run: | echo "## Tests summary" >> $GITHUB_STEP_SUMMARY - sed 's/\[pass\]/:white_check_mark:/g; s/\[\! FAIL \!\]/:x:/g' "${dls_tests_results_path}"/functional_tests_${{ matrix.runner_print_label }}_on_host_${{ matrix.ubuntu_version }}_results.txt >> $GITHUB_STEP_SUMMARY + sed 's/\[pass\]/:white_check_mark:/g; s/\[\! FAIL \!\]/:x:/g' "${dls_tests_results_path}"/DLS_functional_tests_${{ matrix.runner_print_label }}_on_host_${{ matrix.ubuntu_version }}_results.txt >> $GITHUB_STEP_SUMMARY - name: Uninstall dlstreamer if: always () diff --git a/.github/workflows/dls-build-and-test-windows.yaml b/.github/workflows/dls-build-and-test-windows.yaml index adac1a7993..5eee73d923 100644 --- a/.github/workflows/dls-build-and-test-windows.yaml +++ b/.github/workflows/dls-build-and-test-windows.yaml @@ -23,7 +23,7 @@ jobs: runner_print_label: ARL steps: - name: Checkout repository - uses: actions/checkout@v4.2.0 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: path: edge-ai-libraries-repo persist-credentials: false diff --git a/.github/workflows/dls-build-dev-docker-images-and-run-unit.yaml b/.github/workflows/dls-build-dev-docker-images-and-run-unit.yaml index 04664d8f49..d331c4253e 100644 --- a/.github/workflows/dls-build-dev-docker-images-and-run-unit.yaml +++ b/.github/workflows/dls-build-dev-docker-images-and-run-unit.yaml @@ -29,7 +29,7 @@ jobs: sudo rm -rf edge-ai-libraries-repo - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: path: edge-ai-libraries-repo persist-credentials: false @@ -41,10 +41,10 @@ jobs: git submodule update --init libraries/dl-streamer/thirdparty/googletest - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #3.11.1 - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -136,7 +136,7 @@ jobs: env: RESULTS_DIR: test-results with: - name: unit_tests_tgl_${{ matrix.ubuntu_version }} + name: DLS_unit_tests_tgl_${{ matrix.ubuntu_version }} path: ${{ env.RESULTS_DIR }}/*.xml - name: Clean up diff --git a/.github/workflows/dls-build-documentation.yaml b/.github/workflows/dls-build-documentation.yaml index fcae61a1e4..3fe4f5b4c1 100644 --- a/.github/workflows/dls-build-documentation.yaml +++ b/.github/workflows/dls-build-documentation.yaml @@ -19,7 +19,7 @@ jobs: contents: read steps: - name: Check out edge-ai-libraries repository/libraries/dl-streamer - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false sparse-checkout: | diff --git a/.github/workflows/dls-coverity.yaml b/.github/workflows/dls-coverity.yaml index aa920a3415..51386a2b7d 100644 --- a/.github/workflows/dls-coverity.yaml +++ b/.github/workflows/dls-coverity.yaml @@ -21,7 +21,7 @@ jobs: run-analysis: ${{ steps.detect-langs.outputs.run-analysis }} steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: path: edge-ai-libraries-repo persist-credentials: false @@ -61,7 +61,7 @@ jobs: if: needs.detect-languages.outputs.run-analysis == 'true' steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false diff --git a/.github/workflows/dls-download-models.yaml b/.github/workflows/dls-download-models.yaml index c37b883e59..609f8d2e0a 100644 --- a/.github/workflows/dls-download-models.yaml +++ b/.github/workflows/dls-download-models.yaml @@ -67,7 +67,7 @@ jobs: - ${{ matrix.runner }} steps: - name: Get script - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo diff --git a/.github/workflows/dls-pr-workflow.yaml b/.github/workflows/dls-pr-workflow.yaml index 927062d9df..0f49c725e9 100644 --- a/.github/workflows/dls-pr-workflow.yaml +++ b/.github/workflows/dls-pr-workflow.yaml @@ -32,14 +32,15 @@ on: permissions: {} jobs: - code-style: + # ------------------------------------------------------------- SCANs ------------------------------------------------------------- + dls-code-style: permissions: contents: read - name: SCAN code-style + name: "DLS SCAN: code-style" runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false @@ -50,74 +51,63 @@ jobs: uses: ./.github/actions/common/code-style with: target_dir: "libraries/dl-streamer" + name: 'DLS_code-style-check-report' + fail-on-findings: true - check-license-headers: + dls-check-license-headers: permissions: contents: read - name: SCAN check license headers + name: "DLS SCAN: check license headers" runs-on: ubuntu-latest steps: - - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + - name: Check out edge-ai-libraries repository (sparse) + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false + sparse-checkout: | + libraries/dl-streamer + .github + fetch-depth: 0 - name: Check license headers uses: ./.github/actions/common/license-namespace-checker + with: + name: 'DLS_license-check-report' + path: '.' + fail-on-findings: true - # static-code-analysis: + # dls-static-code-analysis: # permissions: # security-events: write # actions: read # contents: read # packages: read - # name: SCAN static code analysis + # name: "DLS SCAN: static code analysis" # uses: ./.github/workflows/codeql.yaml - static-code-analysis-for-c-cpp: + dls-static-code-analysis-for-c-cpp: permissions: security-events: write actions: read contents: read packages: read - name: SCAN DLS static C/C++ code analysis + name: "DLS SCAN: static C/C++ code analysis" uses: ./.github/workflows/dls-coverity.yaml secrets: DLS_COVERITY_TOKEN: ${{ secrets.DLS_COVERITY_TOKEN }} DLS_COVERITY_EMAIL: ${{ secrets.DLS_COVERITY_EMAIL }} DLS_COVERITY_PROJECT: ${{ secrets.DLS_COVERITY_PROJECT }} - build-dev-images-and-run-unit-tests: - permissions: - contents: read - packages: read - name: BUILD dev imgs & run unit tests - uses: ./.github/workflows/dls-build-dev-docker-images-and-run-unit.yaml - - build-and-test-deb-and-deb_img: - permissions: - contents: read - packages: read - name: BUILD & TEST .deb pkgs & img - uses: ./.github/workflows/dls-build-and-test-deb_pkgs-and-deb_imgs.yaml - - build-and-test-windows: + dls-filter-docker-related-changes: permissions: contents: read - packages: read - name: BUILD & TEST Windows DLLs - uses: ./.github/workflows/dls-build-and-test-windows.yaml - - filter-docker-related-changes: - permissions: - contents: read - name: SCAN detect changes in docker dir + name: "DLS SCAN: detect changes in docker dir" runs-on: ubuntu-latest outputs: docker_changed: ${{ steps.check.outputs.docker_changed }} steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false fetch-depth: 0 @@ -125,58 +115,59 @@ jobs: - name: Fetch main branch for comparison run: git fetch origin main - - id: check - name: Detect changes in docker directory + - name: Detect changes in docker directory + id: check run: | echo "πŸ” Checking for changes in 'libraries/dl-streamer/docker/'..." - CHANGED_FILES=$(git diff --name-only origin/main HEAD) - echo "πŸ“„ Changed files:" - echo "$CHANGED_FILES" - if echo "$CHANGED_FILES" | grep -q '^libraries/dl-streamer/docker/'; then + CHANGED_FILES=$(git diff --name-only origin/main...HEAD -- 'libraries/dl-streamer/docker/') + + if [ -n "${CHANGED_FILES}" ]; then + echo "πŸ“„ Changed Docker-related files:" + echo "${CHANGED_FILES}" echo "docker_changed=true" >> "$GITHUB_OUTPUT" echo "🟑 Docker-related changes detected." else - echo "docker_changed=false" >> "$GITHUB_OUTPUT" echo "βœ… No docker-related changes." + echo "docker_changed=false" >> "$GITHUB_OUTPUT" fi - trivy-config-scan: + dls-trivy-config-scan: permissions: contents: read - needs: [filter-docker-related-changes] - if: needs.filter-docker-related-changes.outputs.docker_changed == 'true' - name: SCAN Trivy ${{ matrix.name }} + needs: [dls-filter-docker-related-changes] + if: needs.dls-filter-docker-related-changes.outputs.docker_changed == 'true' + name: "DLS SCAN: Trivy ${{ matrix.name }}" strategy: fail-fast: false matrix: include: - name: dlstreamer_ubuntu22 path: libraries/dl-streamer/docker/ubuntu/ubuntu22.Dockerfile - output: reports/trivy-dlstreamer_ubuntu22.json + output: reports/trivy-DLS_ubuntu22.json - name: dlstreamer_ubuntu24 path: libraries/dl-streamer/docker/ubuntu/ubuntu24.Dockerfile - output: reports/trivy-dlstreamer_ubuntu24.json + output: reports/trivy-DLS_ubuntu24.json - name: dlstreamer_fedora41 path: libraries/dl-streamer/docker/fedora41/fedora41.Dockerfile - output: reports/trivy-dlstreamer_fedora41.json + output: reports/trivy-DLS_fedora41.json - name: dlstreamer-testing-ubuntu path: libraries/dl-streamer/docker/ubuntu/ubuntu-testing.Dockerfile - output: reports/trivy-dlstreamer-testing-ubuntu.json + output: reports/trivy-DLS-testing-ubuntu.json uses: ./.github/workflows/trivy-config-mode.yaml with: dockerfile-path: ${{ matrix.path }} trivy-report-format: 'json' severity-levels: 'HIGH,CRITICAL' output-report-path: ${{ matrix.output }} - name: ${{ matrix.name }} + name: DLS_${{ matrix.name }} - hadolint: + dls-hadolint: permissions: contents: read pull-requests: write - needs: [filter-docker-related-changes] - if: needs.filter-docker-related-changes.outputs.docker_changed == 'true' - name: SCAN Hadolint + needs: [dls-filter-docker-related-changes] + if: needs.dls-filter-docker-related-changes.outputs.docker_changed == 'true' + name: "DLS SCAN: Hadolint" runs-on: ubuntu-latest strategy: fail-fast: false @@ -196,7 +187,7 @@ jobs: output-file: hadolint-dlstreamer-testing-ubuntu.json steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false @@ -205,41 +196,42 @@ jobs: with: dockerfile: ${{ matrix.dockerfile }} output-file: ${{ matrix.output-file }} - name: ${{ matrix.name }} + name: DLS_${{ matrix.name }} enable-reviewdog: true github_token: ${{ secrets.GITHUB_TOKEN }} + fail-on-findings: true - pylint: + dls-pylint: permissions: contents: read pull-requests: write - name: SCAN pylint + name: "DLS SCAN: pylint" runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: - path: edge-ai-libraries-repo persist-credentials: false - name: Run pylint - uses: ./edge-ai-libraries-repo/.github/actions/common/pylint + uses: ./.github/actions/common/pylint with: - path: edge-ai-libraries-repo/libraries/dl-streamer + path: libraries/dl-streamer output-file: pylint-report.txt - name: dls + name: DLS_pylint enable-reviewdog: true github_token: ${{ secrets.GITHUB_TOKEN }} + fail-on-findings: true - shellcheck: + dls-shellcheck: permissions: contents: read pull-requests: write - name: SCAN shellcheck + name: "DLS SCAN: shellcheck" runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: path: edge-ai-libraries-repo persist-credentials: false @@ -250,19 +242,24 @@ jobs: with: path: edge-ai-libraries-repo/libraries/dl-streamer output-file: shellcheck-report.txt - name: dls + name: DLS_shellcheck enable-reviewdog: true github_token: ${{ secrets.GITHUB_TOKEN }} + fail-on-findings: true + + - name: Clean up + run: | + sudo rm -rf edge-ai-libraries-repo - yamllint: + dls-yamllint: permissions: contents: read pull-requests: write - name: SCAN yamllint + name: "DLS SCAN: yamllint" runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: path: edge-ai-libraries-repo persist-credentials: false @@ -272,48 +269,112 @@ jobs: with: path: edge-ai-libraries-repo/libraries/dl-streamer output-file: yamllint-report.txt - name: dls + name: DLS_yamlint enable-reviewdog: true github_token: ${{ secrets.GITHUB_TOKEN }} + fail-on-findings: true - clamav: + - name: Clean up + run: | + sudo rm -rf edge-ai-libraries-repo + + dls-clamav: permissions: contents: read - name: SCAN ClamAV antivirus + name: "DLS SCAN: ClamAV antivirus" runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo - - uses: open-edge-platform/orch-ci/.github/actions/clamav@37eef2d2a0909dfe8ff26bb0730ab2f13dfbcaf6 + - name: ClamAV scan + id: clamav-dls-scan + uses: open-edge-platform/orch-ci/.github/actions/security/clamav@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: - project-folder: edge-ai-libraries-repo/libraries/dl-streamer - report-path: clamav_scan_report-dlstreamer.txt + scan-scope: all + paths: edge-ai-libraries-repo/libraries/dl-streamer + report_suffix: "DLS_ClamAV_antivirus_report" + fail-on-findings: true - - name: Upload ClamAV Scan Report - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: ${{ env.CLAMAV_ARTIFACT_NAME }} - path: ${{ env.SANITIZED_CLAMAV_REPORT_PATH }} + - name: Analyze ClamAV results + if: always() + env: + REPORT_PATH: ${{ steps.clamav-dls-scan.outputs.report_path }} + run: | + if [ -n "$REPORT_PATH" ] && [ -f "$REPORT_PATH" ]; then + echo "πŸ“„ Found ClamAV report: $REPORT_PATH" + + # Extract scan summary using jq + files_scanned=$(jq -r '.scan_summary.files_scanned // 0' "$REPORT_PATH" 2>/dev/null || echo "0") + threats_found=$(jq -r '.scan_summary.threats_found // 0' "$REPORT_PATH" 2>/dev/null || echo "0") + + echo "### ClamAV Antivirus Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- πŸ“ **Files scanned**: $files_scanned" >> $GITHUB_STEP_SUMMARY + echo "- 🦠 **Threats found**: $threats_found" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "$threats_found" -gt 0 ]; then + echo "❌ **Security Alert**: Malware or threats detected!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **Action Required**: Review the ClamAV report artifact for details." >> $GITHUB_STEP_SUMMARY + else + echo "βœ… **All files are clean - no threats detected!**" >> $GITHUB_STEP_SUMMARY + fi + else + echo "### ClamAV Antivirus Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **ClamAV report not found at path: ${REPORT_PATH:-not provided}**" >> $GITHUB_STEP_SUMMARY + fi + + - name: Clean up + if: always() + run: | + sudo rm -rf edge-ai-libraries-repo - bandit: + dls-bandit: permissions: contents: read - name: SCAN Bandit + name: "DLS SCAN: Bandit" runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false - name: Run Bandit scan - uses: open-edge-platform/orch-ci/.github/actions/security/bandit@d475e84f8b0b7bff118f0e8f56712390ef8d2828 + uses: open-edge-platform/orch-ci/.github/actions/security/bandit@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: scan-scope: "changed" severity-level: "HIGH" confidence-level: "HIGH" output-format: "txt" + fail-on-findings: true + paths: libraries/dl-streamer + report_suffix: dlstreamer + + # -------------------------------------------------------- BUILDS & TESTS --------------------------------------------------------- + dls-build-dev-images-and-run-unit-tests: + permissions: + contents: read + packages: read + name: "DLS BUILD: dev imgs & run unit tests" + uses: ./.github/workflows/dls-build-dev-docker-images-and-run-unit.yaml + + dls-build-and-test-deb-and-deb_img: + permissions: + contents: read + packages: read + name: "DLS BUILD & TEST: .deb pkgs & img" + uses: ./.github/workflows/dls-build-and-test-deb_pkgs-and-deb_imgs.yaml + + dls-build-and-test-windows: + permissions: + contents: read + packages: read + name: "DLS BUILD & TEST: Windows DLLs" + uses: ./.github/workflows/dls-build-and-test-windows.yaml diff --git a/.github/workflows/dls-weekly-cached-images.yaml b/.github/workflows/dls-weekly-cached-images.yaml index a6f21c2bdc..248f0f5a18 100644 --- a/.github/workflows/dls-weekly-cached-images.yaml +++ b/.github/workflows/dls-weekly-cached-images.yaml @@ -42,7 +42,7 @@ jobs: ubuntu24_image: ${{ steps.save-image-ubuntu24.outputs.image }} steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false @@ -51,10 +51,10 @@ jobs: git submodule update --init libraries/dl-streamer/thirdparty/spdlog - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #3.11.1 - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -75,9 +75,10 @@ jobs: --build-arg DLSTREAMER_BUILD_NUMBER=deb-pkg-${{ matrix.ubuntu_version }} \ -f ${{ matrix.path_dockerfile }} \ ${{ env.DLS_REL_PATH }} + # ======================================================== SCANNING PART ======================================================== - name: πŸ” Scan Docker image with Trivy - uses: open-edge-platform/orch-ci/.github/actions/security/trivy@65fc743e0c69b8529188b89dede3acfe2897b2e0 + uses: open-edge-platform/orch-ci/.github/actions/security/trivy@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: scan_target: "ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer:${{ inputs.image-tag }}-${{ matrix.ubuntu_version }}" severity: "HIGH" @@ -126,14 +127,14 @@ jobs: IMAGE=ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer:${IMAGE_TAG}-${{ matrix.ubuntu_version }} DIGEST=$(skopeo inspect docker://$IMAGE | jq -r '.Digest') echo "digest=${DIGEST}" >> $GITHUB_OUTPUT - + - name: Sign Docker image using Cosign (keyless) if: ${{ inputs.action-type == 'weekly' }} env: deb_final_img: ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer@${{ steps.digest.outputs.digest }} run: | cosign sign --yes ${deb_final_img} - + - name: Save Ubuntu 22 image info id: save-image-ubuntu22 if: ${{ matrix.ubuntu_version == 'ubuntu22' }} @@ -175,7 +176,7 @@ jobs: build_arg: Debug steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false @@ -184,10 +185,10 @@ jobs: git submodule update --init libraries/dl-streamer/thirdparty/spdlog - name: Set up Docker Buildx - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 #3.10.0 + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #3.11.1 - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -207,9 +208,10 @@ jobs: --build-arg BUILD_ARG=${{ matrix.build_arg }} \ -f ${{ matrix.path_dockerfile }} \ ./libraries/dl-streamer + # ======================================================== SCANNING PART ======================================================== - name: Scan Docker image with Trivy - uses: open-edge-platform/orch-ci/.github/actions/security/trivy@65fc743e0c69b8529188b89dede3acfe2897b2e0 + uses: open-edge-platform/orch-ci/.github/actions/security/trivy@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: scan_target: "ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer:${{ inputs.image-tag }}-dev-${{ matrix.ubuntu_version }}" severity: "HIGH" @@ -240,7 +242,7 @@ jobs: IMAGE_TAG: ${{ inputs.image-tag }} run: | docker push "ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer:${IMAGE_TAG}-dev-${{ matrix.ubuntu_version }}" - + - name: Install Cosign if: ${{ inputs.action-type == 'weekly' }} uses: sigstore/cosign-installer@398d4b0eeef1380460a10c8013a76f728fb906ac # v3.9.1 @@ -271,6 +273,8 @@ jobs: run: | rm -rf edge-ai-libraries-repo docker rmi ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer:${IMAGE_TAG}-dev-${{ matrix.ubuntu_version }} || true + + build-dls-pipeline-server-image: if: ${{ inputs.action-type == 'weekly' }} needs: build-dls-deb-img @@ -286,7 +290,7 @@ jobs: ubuntu_version: [ubuntu22, ubuntu24] steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo @@ -298,7 +302,7 @@ jobs: git submodule update --init libraries/dl-streamer/dl-streamer-tests - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -325,9 +329,10 @@ jobs: export DLSTREAMER_PIPELINE_SERVER_IMAGE=ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer-pipeline-server:${IMAGE_TAG}-extended-${{ matrix.ubuntu_version }} export BUILD_TARGET=dlstreamer-pipeline-server-extended docker compose build --no-cache --pull + # ======================================================== SCANNING PART ======================================================== - name: Scan Docker image with Trivy - uses: open-edge-platform/orch-ci/.github/actions/security/trivy@65fc743e0c69b8529188b89dede3acfe2897b2e0 + uses: open-edge-platform/orch-ci/.github/actions/security/trivy@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: scan_target: "ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer-pipeline-server:${{ inputs.image-tag }}-${{ matrix.ubuntu_version }}" severity: "HIGH" @@ -338,7 +343,7 @@ jobs: timeout: 20m ignore_unfixed: true - name: Scan Docker Extended image with Trivy - uses: open-edge-platform/orch-ci/.github/actions/security/trivy@65fc743e0c69b8529188b89dede3acfe2897b2e0 + uses: open-edge-platform/orch-ci/.github/actions/security/trivy@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: scan_target: "ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer-pipeline-server:${{ inputs.image-tag }}-extended-${{ matrix.ubuntu_version }}" severity: "HIGH" @@ -397,6 +402,7 @@ jobs: DIGEST_LATEST=$(skopeo inspect docker://ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer-pipeline-server:latest | jq -r '.Digest') echo "digest_latest=$DIGEST_LATEST" >> $GITHUB_OUTPUT fi + - name: Sign Docker image using Cosign (keyless) env: dlsps_img: ghcr.io/${{ github.repository }}/intel/edge-ai-dlstreamer-pipeline-server@${{ steps.digest.outputs.digest }} @@ -404,6 +410,7 @@ jobs: run: | cosign sign --yes ${dlsps_img} cosign sign --yes ${dlsps_img_ext} + - name: Sign Docker image using Cosign (keyless) (latest) if: ${{ matrix.ubuntu_version == 'ubuntu24' }} env: diff --git a/.github/workflows/dlsps-build-scans-pr-workflow.yaml b/.github/workflows/dlsps-build-scans-pr-workflow.yaml index 4df49f5c43..ff68f0c00b 100644 --- a/.github/workflows/dlsps-build-scans-pr-workflow.yaml +++ b/.github/workflows/dlsps-build-scans-pr-workflow.yaml @@ -1,23 +1,17 @@ -name: "[DLSPS] PR workflow" -run-name: "[DLSPS] PR workflow (by @${{ github.actor }} via ${{ github.event_name }})" +name: "[DLSPS] Build, scan and test" +run-name: "[DLSPS] Build, scan and test (by @${{ github.actor }} via ${{ github.event_name }})" on: - push: - branches: - - 'main' - paths: - - 'microservices/dlstreamer-pipeline-server/**' - - pull_request: - paths: - - 'microservices/dlstreamer-pipeline-server/**' - workflow_call: + workflow_dispatch: permissions: {} +env: + dlstreamer-version: "2025.1.2" + DLS_REL_PATH: "./edge-ai-libraries-repo/libraries/dl-streamer" jobs: build-dls-pipeline-server-image: name: Build DLS Pipeline Server ${{ matrix.ubuntu_version }} img - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-16core-64GB permissions: contents: read packages: write @@ -29,28 +23,42 @@ jobs: - ubuntu_version: ubuntu24 steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo + - name: Init submodules + run: | + cd edge-ai-libraries-repo + git submodule update --init libraries/dl-streamer/thirdparty/spdlog + - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set BASE_IMAGE + - name: Build DL Streamer deb-final-img env: - BASE_IMAGE22: "ghcr.io/open-edge-platform/edge-ai-libraries/intel/edge-ai-dlstreamer:20250805_EAL1.2-ubuntu22" - BASE_IMAGE24: "ghcr.io/open-edge-platform/edge-ai-libraries/intel/edge-ai-dlstreamer:20250805_EAL1.2-ubuntu24" + deb_final_img: dlstreamer-base:${{ matrix.ubuntu_version }}-${{ github.sha }} + deb_final_img_cached: ghcr.io/${{ github.repository }}/deb-final-img-${{ matrix.ubuntu_version }}:buildcache + dls_ubuntu22_dockefile: ${{ env.DLS_REL_PATH }}/docker/ubuntu/ubuntu22.Dockerfile + dls_ubuntu24_dockefile: ${{ env.DLS_REL_PATH }}/docker/ubuntu/ubuntu24.Dockerfile run: | - if [ "${{ matrix.ubuntu_version }}" == "ubuntu22" ]; then - echo "BASE_IMAGE=${BASE_IMAGE22}" >> $GITHUB_ENV - elif [ "${{ matrix.ubuntu_version }}" == "ubuntu24" ]; then - echo "BASE_IMAGE=${BASE_IMAGE24}" >> $GITHUB_ENV - fi + DLS_PATH="${{ env.DLS_REL_PATH }}" + docker pull "${deb_final_img_cached}" || true + docker buildx build \ + --load \ + --target dlstreamer \ + --tag "${deb_final_img}" \ + --cache-from="${deb_final_img_cached}" \ + --build-arg DLSTREAMER_VERSION=${{ env.dlstreamer-version }} \ + --build-arg DLSTREAMER_BUILD_NUMBER=deb-pkg-${{ matrix.ubuntu_version }} \ + -f "${DLS_PATH}/docker/ubuntu/${{ matrix.ubuntu_version }}.Dockerfile" \ + "${DLS_PATH}" + echo "BASE_IMAGE=${deb_final_img}" >> $GITHUB_ENV - name: Build dls-pipeline-server-img run: | @@ -58,21 +66,23 @@ jobs: export DLSTREAMER_PIPELINE_SERVER_IMAGE=intel/dlstreamer-pipeline-server:3.1.0-${{ matrix.ubuntu_version }} export DLSTREAMER_PIPELINE_SERVER_DOCKERFILE=Dockerfile export BUILD_TARGET=dlstreamer-pipeline-server - docker compose build --no-cache --pull + docker compose build --no-cache + - name: Build dls-pipeline-server-img-extended run: | cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/docker export DLSTREAMER_PIPELINE_SERVER_IMAGE=intel/dlstreamer-pipeline-server:3.1.0-extended-${{ matrix.ubuntu_version }} export DLSTREAMER_PIPELINE_SERVER_DOCKERFILE=Dockerfile - BUILD_TARGET=dlstreamer-pipeline-server-extended - docker compose build --no-cache --pull + export BUILD_TARGET=dlstreamer-pipeline-server-extended + docker compose build --no-cache + - name: Unit Test dls-pipeline-server if: matrix.ubuntu_version == 'ubuntu22' run: | cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/ make build make test | tee /tmp/pytest_output.txt - exit ${PIPESTATUS[0]} + - name: Create Unit Test Summary if: matrix.ubuntu_version == 'ubuntu22' run: | @@ -82,54 +92,150 @@ jobs: echo "### Pytest Summary" >> "$GITHUB_STEP_SUMMARY" echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" echo "$summary_line" >> "$GITHUB_STEP_SUMMARY" - echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY" + - name: Upload unit test results to Github if: matrix.ubuntu_version == 'ubuntu22' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: Coverage-reports - path: /tmp/htmlcov + name: DLSPS_Coverage-reports + path: /tmp/htmlcov + - name: Scan Docker image with Trivy - uses: ./edge-ai-libraries-repo/.github/actions/common/trivy-image-scan + uses: open-edge-platform/orch-ci/.github/actions/security/trivy@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: - image: "intel/dlstreamer-pipeline-server:3.1.0-${{ matrix.ubuntu_version }}" - severity: "CRITICAL" - # output-format: "json" - output-file: "dlsps-${{ matrix.ubuntu_version }}-trivy-image-report.txt" + scan_target: "intel/dlstreamer-pipeline-server:3.1.0-${{ matrix.ubuntu_version }}" + severity: "HIGH" + scan_type: image + format: table + report_suffix: "-${{ matrix.ubuntu_version }}-dlsps-img" + scan-scope: all + timeout: 20m + ignore_unfixed: true + + - name: Rename Trivy report for base image + run: | + echo "πŸ“ Renaming Trivy report for base image..." + if [ -d "security-results/trivy" ]; then + latest=$(ls -t security-results/trivy/trivy-results-*.table 2>/dev/null | head -n 1) + if [ -n "$latest" ]; then + mv "$latest" "security-results/trivy/trivy-base-${{ matrix.ubuntu_version }}.table" + echo "βœ… Renamed to: trivy-base-${{ matrix.ubuntu_version }}.table" + else + echo "⚠️ No .table file found to rename" + fi + fi + shell: bash + - name: Scan Docker extended image with Trivy - uses: ./edge-ai-libraries-repo/.github/actions/common/trivy-image-scan + uses: open-edge-platform/orch-ci/.github/actions/security/trivy@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: - image: "intel/dlstreamer-pipeline-server:3.1.0-extended-${{ matrix.ubuntu_version }}" - severity: "CRITICAL" - # output-format: "json" - output-file: "dlsps-${{ matrix.ubuntu_version }}-extended-trivy-image-report.txt" + scan_target: "intel/dlstreamer-pipeline-server:3.1.0-extended-${{ matrix.ubuntu_version }}" + severity: "HIGH" + scan_type: image + format: table + report_suffix: "-${{ matrix.ubuntu_version }}-dlsps-extended-img" + scan-scope: all + timeout: 20m + ignore_unfixed: true + + - name: Rename Trivy report for extended image + run: | + echo "πŸ“ Renaming Trivy report for extended image..." + if [ -d "security-results/trivy" ]; then + latest=$(ls -t security-results/trivy/trivy-results-*.table 2>/dev/null | head -n 1) + if [ -n "$latest" ]; then + mv "$latest" "security-results/trivy/trivy-extended-${{ matrix.ubuntu_version }}.table" + echo "βœ… Renamed to: trivy-extended-${{ matrix.ubuntu_version }}.table" + else + echo "⚠️ No .table file found to rename" + fi + fi + shell: bash + - name: Upload Trivy image report as artifact if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #4.6.2 with: - name: dlsps-${{ matrix.ubuntu_version }}-trivy-image-report - path: dlsps-${{ matrix.ubuntu_version }}*-trivy-image-report.txt + name: DLSPS_${{ matrix.ubuntu_version }}-trivy-image-reports + path: | + security-results/trivy/trivy-base-${{ matrix.ubuntu_version }}.table + security-results/trivy/trivy-extended-${{ matrix.ubuntu_version }}.table - name: Run Trivy Filesystem Scan if: matrix.ubuntu_version == 'ubuntu22' run: | - docker pull aquasec/trivy:0.63.0 cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/ mkdir -p reports curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/html.tpl -o trivy-html.tpl - - docker run --rm -v `pwd`:/src aquasec/trivy:0.63.0 fs /src/ --format template --template "@/src/trivy-html.tpl" -o "/src/reports/trivy_fs_code_scan.html" || true + + docker run --rm -v `pwd`:/src aquasec/trivy:0.63.0 fs /src/ --format template --template "@/src/trivy-html.tpl" -o "/src/reports/trivy_fs_code_scan.html" || true docker run --rm -v `pwd`:/src aquasec/trivy:0.63.0 fs --list-all-pkgs --format template --template "@/src/trivy-html.tpl" --output "/src/reports/trivy-fs-full-report.csv" /src/ || true docker run --rm -v `pwd`:/src aquasec/trivy:0.63.0 fs --ignore-unfixed /src | tee ./reports/trivy-fs-full-report.txt mv ./reports ${{ github.workspace }} + - name: Upload Trivy Filesystem Reports if: matrix.ubuntu_version == 'ubuntu22' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 #4.6.2 with: - name: trivy-fs-reports - path: reports/* - + name: DLSPS_trivy-fs-reports + path: reports/* + + - name: Analyze Trivy results and create summary + if: always() + run: | + echo "### πŸ” Trivy Security Scan Results for ${{ matrix.ubuntu_version }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + base_file="security-results/trivy/trivy-base-${{ matrix.ubuntu_version }}.table" + extended_file="security-results/trivy/trivy-extended-${{ matrix.ubuntu_version }}.table" + + # Function to count vulnerabilities + count_vulns() { + local file=$1 + if [ -f "$file" ]; then + awk '/β”‚/ && /Vulnerabilities/ {next} /β”‚/ {gsub(/ /, "", $0); split($0, cols, "β”‚"); print cols[4]}' "$file" | grep -v '^-$' | grep -v '^$' | head -n 1 + else + echo "N/A" + fi + } + + base_vulns=$(count_vulns "$base_file") + extended_vulns=$(count_vulns "$extended_file") + + # Base image results + echo "#### πŸ“¦ Base Image: \`intel/dlstreamer-pipeline-server:3.1.0-${{ matrix.ubuntu_version }}\`" >> $GITHUB_STEP_SUMMARY + if [ -f "$base_file" ]; then + if [ "$base_vulns" = "0" ] || [ -z "$base_vulns" ]; then + echo "- βœ… **No HIGH vulnerabilities found**" >> $GITHUB_STEP_SUMMARY + else + echo "- ❌ **Vulnerabilities found:** $base_vulns" >> $GITHUB_STEP_SUMMARY + fi + else + echo "- ⚠️ **Report not found**" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + + # Extended image results + echo "#### πŸ“¦ Extended Image: \`intel/dlstreamer-pipeline-server:3.1.0-extended-${{ matrix.ubuntu_version }}\`" >> $GITHUB_STEP_SUMMARY + if [ -f "$extended_file" ]; then + if [ "$extended_vulns" = "0" ] || [ -z "$extended_vulns" ]; then + echo "- βœ… **No HIGH vulnerabilities found**" >> $GITHUB_STEP_SUMMARY + else + echo "- ❌ **Vulnerabilities found:** $extended_vulns" >> $GITHUB_STEP_SUMMARY + fi + else + echo "- ⚠️ **Report not found**" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY + echo "---" >> $GITHUB_STEP_SUMMARY + + # Store results for final check + echo "BASE_VULNS=$base_vulns" >> $GITHUB_ENV + echo "EXTENDED_VULNS=$extended_vulns" >> $GITHUB_ENV + shell: bash + - name: Create summary if: always() run: | @@ -139,133 +245,240 @@ jobs: echo "intel/dlstreamer-pipeline-server:3.1.0-extended-${{ matrix.ubuntu_version }}" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo "Built on commit id: ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY + + - name: Fail if vulnerabilities found + if: always() + run: | + base_vulns="${BASE_VULNS:-N/A}" + extended_vulns="${EXTENDED_VULNS:-N/A}" + + echo "πŸ“Š Final vulnerability check:" + echo " Base image: $base_vulns" + echo " Extended image: $extended_vulns" + + if [[ "$base_vulns" != "0" && "$base_vulns" != "N/A" && -n "$base_vulns" ]] || \ + [[ "$extended_vulns" != "0" && "$extended_vulns" != "N/A" && -n "$extended_vulns" ]]; then + echo "❌ Vulnerabilities detected in one or more images!" + exit 1 + else + echo "βœ… All security scans passed!" + fi + shell: bash + - name: Clean up if: always() run: | - rm -rf edge-ai-libraries-repo - sudo rm -rf /tmp/htmlcov + sudo rm -rf edge-ai-libraries-repo + sudo rm -rf /tmp/htmlcov reports/* docker rmi intel/dlstreamer-pipeline-server:3.1.0-${{ matrix.ubuntu_version }} || true docker rmi intel/dlstreamer-pipeline-server:3.1.0-extended-${{ matrix.ubuntu_version }} || true - - bandit-virus-scans: - name: Run Bandit and Virus Scan + + dlsps-code-style: + permissions: + contents: read + name: "DLSPS SCAN: code-style" + runs-on: ubuntu-latest + steps: + - name: Check out edge-ai-libraries repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + with: + persist-credentials: false + + - name: Code-style action + uses: ./.github/actions/common/code-style + with: + target_dir: "microservices/dlstreamer-pipeline-server" + name: 'DLSPS_code-style-check-report' + fail-on-findings: true + + dlsps-check-license-headers: + permissions: + contents: read + name: "DLSPS SCAN: check license headers" + runs-on: ubuntu-latest + steps: + - name: Check out edge-ai-libraries repository (sparse) + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + with: + persist-credentials: false + sparse-checkout: | + microservices/dlstreamer-pipeline-server + .github + fetch-depth: 0 + + - name: Check license headers + uses: ./.github/actions/common/license-namespace-checker + with: + name: 'DLSPS_license-check-report' + path: '.' + fail-on-findings: true + + dlsps-clamav: + permissions: + contents: read + name: "DLSPS SCAN: ClamAV antivirus" runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - ubuntu_version: ubuntu22 steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo - - name: Run Bandit Scan - run: | - mkdir -p reports - docker pull ghcr.io/pycqa/bandit/bandit - echo "### Bandit Scan Results" >> $GITHUB_STEP_SUMMARY - docker run --rm -v "${{ github.workspace }}:/src" ghcr.io/pycqa/bandit/bandit -r /src/edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server -f txt -o /src/reports/bandit-report.txt || true >> $GITHUB_STEP_SUMMARY - echo "Please find full report in bandit-report.txt" >> $GITHUB_STEP_SUMMARY - - name: Run Virus Scan - run: | - mkdir -p reports - docker pull clamav/clamav - echo "### Virus Scan Results" >> $GITHUB_STEP_SUMMARY - docker run --rm -v "${{ github.workspace }}:/src" clamav/clamav clamscan -r /src/edge-ai-libraries/microservices/dlstreamer-pipeline-server/ > ./reports/clamav-report.txt || true - echo "Please find full report in clamav-report.txt" >> $GITHUB_STEP_SUMMARY - - name: Upload Scan Reports - uses: actions/upload-artifact@v4 + - name: ClamAV scan + id: clamav-dlsps-scan + uses: open-edge-platform/orch-ci/.github/actions/security/clamav@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 with: - name: bandit-virus-reports - path: reports/ - - name: Clean up + scan-scope: all + paths: edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server + report_suffix: "DLSPS_ClamAV_antivirus_report" + fail-on-findings: true + + - name: Analyze ClamAV results if: always() + env: + REPORT_PATH: ${{ steps.clamav-dlsps-scan.outputs.report_path }} run: | - rm -rf edge-ai-libraries-repo - if [ -n "$(docker images -aq)" ]; then - docker rmi -f $(docker images -aq) || true + if [ -n "$REPORT_PATH" ] && [ -f "$REPORT_PATH" ]; then + echo "πŸ“„ Found ClamAV report: $REPORT_PATH" + + # Extract scan summary using jq + files_scanned=$(jq -r '.scan_summary.files_scanned // 0' "$REPORT_PATH" 2>/dev/null || echo "0") + threats_found=$(jq -r '.scan_summary.threats_found // 0' "$REPORT_PATH" 2>/dev/null || echo "0") + + echo "### ClamAV Antivirus Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- πŸ“ **Files scanned**: $files_scanned" >> $GITHUB_STEP_SUMMARY + echo "- 🦠 **Threats found**: $threats_found" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ "$threats_found" -gt 0 ]; then + echo "❌ **Security Alert**: Malware or threats detected!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **Action Required**: Review the ClamAV report artifact for details." >> $GITHUB_STEP_SUMMARY + else + echo "βœ… **All files are clean - no threats detected!**" >> $GITHUB_STEP_SUMMARY + fi + else + echo "### ClamAV Antivirus Scan Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "⚠️ **ClamAV report not found at path: ${REPORT_PATH:-not provided}**" >> $GITHUB_STEP_SUMMARY fi - + - name: Clean up + if: always() + run: | + sudo rm -rf edge-ai-libraries-repo + + dlsps-bandit: + permissions: + contents: read + name: "DLSPS SCAN: Bandit" + runs-on: ubuntu-latest + steps: + - name: Check out edge-ai-libraries repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + with: + persist-credentials: false + + - name: Run Bandit scan + uses: open-edge-platform/orch-ci/.github/actions/security/bandit@76700c2fb6d547733b9218d9638dca43f5296399 # 0.1.52 + with: + scan-scope: "changed" + severity-level: "HIGH" + confidence-level: "HIGH" + output-format: "txt" + paths: microservices/dlstreamer-pipeline-server + report_suffix: dlsps + fail-on-findings: true + trivy-config-dockerfile-scan: permissions: contents: read - name: Scan Dockerfile + name: "DLSPS SCAN: Trivy Dockerfile" strategy: fail-fast: false uses: ./.github/workflows/trivy-config-mode.yaml with: - dockerfile-path: microservices/dlstreamer-pipeline-server/ - trivy-report-format: 'table' + dockerfile-path: microservices/dlstreamer-pipeline-server + trivy-report-format: 'json' severity-levels: 'HIGH,CRITICAL' - output-report-path: reports/dlsps_trivy_config_report.txt - name: dlsps_trivy_report - - pylint: - runs-on: ubuntu-latest + output-report-path: reports/DLSPS_trivy_config_report.txt + name: DLSPS_trivy_report + dlsps-pylint: + permissions: + contents: read + pull-requests: write + name: "DLSPS SCAN: pylint" + runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false - path: edge-ai-libraries-repo - - name: Set up Python - uses: actions/setup-python@v5 + - name: Run pylint + uses: ./.github/actions/common/pylint with: - python-version: '3.10' + path: microservices/dlstreamer-pipeline-server + output-file: pylint-report.txt + name: DLSPS_pylint + enable-reviewdog: true + github_token: ${{ secrets.GITHUB_TOKEN }} + fail-on-findings: true - - name: Install dependencies from all requirements.txt files - run: | - python -m pip install --upgrade pip - cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/ - find . -type f -iname 'requirements.txt' -exec pip install -r {} \; + dlsps-shellcheck: + permissions: + contents: read + pull-requests: write + name: "DLSPS SCAN: shellcheck" + runs-on: ubuntu-latest + steps: + - name: Check out edge-ai-libraries repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 + with: + path: edge-ai-libraries-repo + persist-credentials: false + fetch-depth: 0 - - name: Install pylint - run: pip install pylint + - name: Run shellcheck + uses: ./edge-ai-libraries-repo/.github/actions/common/shellcheck + with: + path: edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server + output-file: shellcheck-report.txt + name: DLSPS_shellcheck + enable-reviewdog: true + github_token: ${{ secrets.GITHUB_TOKEN }} + fail-on-findings: true - - name: Run pylint + - name: Clean up run: | - cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server - find . -type f -iname '*.py' -exec pylint --errors-only --disable=import-error {} \; > pylint_report.txt || true - cp pylint_report.txt ${{ github.workspace }} - echo "### Pylint Results" >> $GITHUB_STEP_SUMMARY - echo "Please find pylint report in pylint-report.txt" >> $GITHUB_STEP_SUMMARY - - - name: Upload pylint report as artifact - uses: actions/upload-artifact@v4 - with: - name: pylint-report - path: pylint_report.txt - shellcheck: - runs-on: ubuntu-latest + sudo rm -rf edge-ai-libraries-repo + dlsps-yamllint: + permissions: + contents: read + pull-requests: write + name: "DLSPS SCAN: yamllint" + runs-on: ubuntu-latest steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: - persist-credentials: false path: edge-ai-libraries-repo + persist-credentials: false - - name: Install ShellCheck - run: sudo apt-get update && sudo apt-get install -y shellcheck + - name: Run yamlint + uses: ./edge-ai-libraries-repo/.github/actions/common/yamllint + with: + path: edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server + output-file: yamllint-report.txt + name: DLSPS_yamlint + enable-reviewdog: true + github_token: ${{ secrets.GITHUB_TOKEN }} + fail-on-findings: true - - name: Run ShellCheck + - name: Clean up run: | - cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/ - echo "Scanning for .sh files and running ShellCheck..." - find . -type f -name "*.sh" | tee shell_files.txt | xargs -r shellcheck -f gcc > shellcheck_report.txt || true - echo "### ShellCheck Results" >> $GITHUB_STEP_SUMMARY - echo "Please find ShellCheck report in shellcheck_report.txt" >> $GITHUB_STEP_SUMMARY - cp shellcheck_report.txt ${{ github.workspace }} - - - name: Upload ShellCheck report - uses: actions/upload-artifact@v4 - with: - name: shellcheck-report - path: shellcheck_report.txt - \ No newline at end of file + sudo rm -rf edge-ai-libraries-repo diff --git a/.github/workflows/dlsps-package-helm-weekly.yaml b/.github/workflows/dlsps-package-helm-weekly.yaml index 8570bd5137..4db11b7033 100644 --- a/.github/workflows/dlsps-package-helm-weekly.yaml +++ b/.github/workflows/dlsps-package-helm-weekly.yaml @@ -8,7 +8,7 @@ on: description: 'Helm chart tag' required: true type: string - + permissions: {} jobs: publish-helm: @@ -21,16 +21,16 @@ jobs: fail-fast: false steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: path: edge-ai-libraries-repo persist-credentials: false - name: Install Helm uses: azure/setup-helm@v4 with: - version: v3.15.2 + version: v3.15.2 - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -38,19 +38,19 @@ jobs: - name: Package Helm Chart run: | - cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/helm + cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/helm helm package . --version ${{ inputs.helm-chart-tag }} --app-version ${{ inputs.helm-chart-tag }} - name: Push to GHCR run: | CHART_PACKAGE=$(ls edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/helm/dlstreamer-pipeline-server-${{ inputs.helm-chart-tag }}.tgz) helm push $CHART_PACKAGE oci://ghcr.io/${{ github.repository }}/ - + - name: Update Github Summary run: | echo "### βœ… DLStreamerPipelineServer helm chart published to github container registry" >> $GITHUB_STEP_SUMMARY echo "- Registry: \`oci://ghcr.io/${{ github.repository }}\`" >> $GITHUB_STEP_SUMMARY echo "- Version: \`${{ inputs.helm-chart-tag }}\`" >> $GITHUB_STEP_SUMMARY - echo "- Pull command: \`helm pull oci://ghcr.io/${{ github.repository }}/dlstreamer-pipeline-server --version ${{ inputs.helm-chart-tag }}\`" >> $GITHUB_STEP_SUMMARY + echo "- Pull command: \`helm pull oci://ghcr.io/${{ github.repository }}/dlstreamer-pipeline-server --version ${{ inputs.helm-chart-tag }}\`" >> $GITHUB_STEP_SUMMARY - name: Clean up if: always() run: | diff --git a/.github/workflows/dlsps-pr-workflow.yaml b/.github/workflows/dlsps-pr-workflow.yaml new file mode 100644 index 0000000000..8cd1e64d96 --- /dev/null +++ b/.github/workflows/dlsps-pr-workflow.yaml @@ -0,0 +1,63 @@ +name: "[DLSPS] PR workflow" +run-name: "[DLSPS] PR workflow (by @${{ github.actor }} via ${{ github.event_name }})" +on: + push: + branches: + - 'main' + paths: + - 'libraries/dl-streamer/*' + - 'libraries/dl-streamer/cmake/**' + - 'libraries/dl-streamer/docker/**' + - 'libraries/dl-streamer/include/**' + - 'libraries/dl-streamer/python/**' + - 'libraries/dl-streamer/samples/**' + - 'libraries/dl-streamer/scripts/**' + - 'libraries/dl-streamer/src/**' + - 'libraries/dl-streamer/tests/**' + - 'libraries/dl-streamer/thirdparty/**' + - 'microservices/dlstreamer-pipeline-server/**' + pull_request: + paths: + - 'libraries/dl-streamer/*' + - 'libraries/dl-streamer/cmake/**' + - 'libraries/dl-streamer/docker/**' + - 'libraries/dl-streamer/include/**' + - 'libraries/dl-streamer/python/**' + - 'libraries/dl-streamer/samples/**' + - 'libraries/dl-streamer/scripts/**' + - 'libraries/dl-streamer/src/**' + - 'libraries/dl-streamer/tests/**' + - 'libraries/dl-streamer/thirdparty/**' + - 'microservices/dlstreamer-pipeline-server/**' + workflow_call: + workflow_dispatch: +permissions: {} + +jobs: + # ----------------------------------------------------- SCANs ----------------------------------------------------- + dlsps-coverity: + permissions: + contents: read + packages: write + name: "DLSPS SCAN: static C/C++ code analysis" + uses: ./.github/workflows/dlsps_coverity.yaml + secrets: + DLSPS_COVERITY_TOKEN: ${{ secrets.DLSPS_COVERITY_TOKEN }} + DLSPS_COVERITY_EMAIL: ${{ secrets.DLSPS_COVERITY_EMAIL }} + DLSPS_COVERITY_PROJECT: ${{ secrets.DLSPS_COVERITY_PROJECT }} + + # ------------------------------------------------ BUILDS & TESTS ------------------------------------------------- + dlsps-sanity-test: + permissions: + contents: read + packages: write + name: "DLSPS TEST: sanity tests" + uses: ./.github/workflows/dlsps_sanity_test.yaml + + dlsps-build-scans-tests: + permissions: + contents: read + packages: write + pull-requests: write + name: "DLSPS BUILD & SCAN & TEST" + uses: ./.github/workflows/dlsps-build-scans-pr-workflow.yaml diff --git a/.github/workflows/dlsps_coverity.yaml b/.github/workflows/dlsps_coverity.yaml index aa07aad0ea..aa8a3ecb8b 100644 --- a/.github/workflows/dlsps_coverity.yaml +++ b/.github/workflows/dlsps_coverity.yaml @@ -1,45 +1,44 @@ name: "[DLSPS] Coverity workflow for C/C++" run-name: "[DLSPS] Coverity scan (by @${{ github.actor }} via ${{ github.event_name }})" on: - push: - branches: - - 'main' - paths: - - 'microservices/dlstreamer-pipeline-server/**' - - pull_request: - paths: - - 'microservices/dlstreamer-pipeline-server/**' - workflow_call: + secrets: + DLSPS_COVERITY_TOKEN: + required: true + DLSPS_COVERITY_EMAIL: + required: true + DLSPS_COVERITY_PROJECT: + required: true + workflow_dispatch: permissions: {} - jobs: coverity-scan: runs-on: ubuntu-latest permissions: contents: read packages: write - + strategy: fail-fast: false matrix: include: - ubuntu_version: ubuntu22 - + steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo + - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build dls-pipeline-server-img run: | cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/docker @@ -48,24 +47,18 @@ jobs: export BASE_IMAGE="ghcr.io/open-edge-platform/edge-ai-libraries/intel/edge-ai-dlstreamer:20250805_EAL1.2-ubuntu22" export BUILD_TARGET=gstudfloader-builder docker compose build --no-cache --pull - - - name: Run Coverity Scans. + + - name: Run Coverity Scans run: | cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/tests echo DLSPS_COVERITY_TOKEN=${{secrets.DLSPS_COVERITY_TOKEN}} >> .env echo DLSPS_COVERITY_EMAIL=${{secrets.DLSPS_COVERITY_EMAIL}} >> .env echo DLSPS_COVERITY_PROJECT=${{secrets.DLSPS_COVERITY_PROJECT}} >> .env docker run --rm --env-file .env -v `pwd`:/app -v /tmp:/tmp intel/dlstreamer-pipeline-server:coverity-ubuntu22 /bin/bash /app/coverity.sh - - name: Upload Coverity Reports to Github - - uses: actions/upload-artifact@v4 - with: - name: Coverity-reports - path: /tmp/coverity-output.tgz + - name: Clean up if: always() run: | rm -rf edge-ai-libraries-repo sudo rm -rf /tmp/coverity-output.tgz docker rmi intel/dlstreamer-pipeline-server:coverity-${{ matrix.ubuntu_version }} || true - \ No newline at end of file diff --git a/.github/workflows/dlsps_sanity_test.yaml b/.github/workflows/dlsps_sanity_test.yaml index 160f7e52ff..03994a1f09 100644 --- a/.github/workflows/dlsps_sanity_test.yaml +++ b/.github/workflows/dlsps_sanity_test.yaml @@ -1,21 +1,12 @@ name: "[DLSPS] PR Sanity workflow" run-name: "[DLSPS] PR Sanity workflow (by @${{ github.actor }} via ${{ github.event_name }})" on: - push: - branches: - - 'main' - paths: - - 'microservices/dlstreamer-pipeline-server/**' - - pull_request: - paths: - - 'microservices/dlstreamer-pipeline-server/**' - workflow_call: + workflow_dispatch: permissions: {} + jobs: sanity: - runs-on: ubuntu-latest permissions: contents: read @@ -25,26 +16,27 @@ jobs: matrix: include: - ubuntu_version: ubuntu22 - steps: - name: Check out edge-ai-libraries repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #4.2.2 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #5.0.0 with: persist-credentials: false path: edge-ai-libraries-repo + - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.10' + - name: Log in to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 #3.4.0 + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #3.6.0 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Sanity Test - - run: | + run: | sudo apt-get update sudo pip install robotframework sudo apt install -y python3-nose libxml2-utils vlc @@ -57,29 +49,48 @@ jobs: sleep 10 cd ${{github.workspace}} cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/tests/scripts/robot_files - robot test_main_sanity.robot || true + robot test_main_sanity.robot || true mkdir -p /tmp/test_results cp -r report.html log.html output.xml /tmp/test_results/ passed=$(xmllint --xpath "//return/status[@status='PASS']" ./output.xml | wc -l) || true failed=$(xmllint --xpath "//return/status[@status='FAIL']" ./output.xml | wc -l) || true not_run=$(xmllint --xpath "//return/status[@status='NOT RUN']" ./output.xml | wc -l) || true total=$((passed + failed + not_run)) + executed=$((passed + failed)) + pass_rate="N/A" + execution_rate="N/A" + if [ "$executed" -gt 0 ]; then + pass_rate=$(awk -v p="$passed" -v e="$executed" 'BEGIN { printf "%.2f%%", (p / e) * 100 }') + fi + if [ "$total" -gt 0 ]; then + execution_rate=$(awk -v e="$executed" -v t="$total" 'BEGIN { printf "%.2f%%", (e / t) * 100 }') + fi echo "### Sanity Test Summary" >> $GITHUB_STEP_SUMMARY echo "- Total: $total" >> $GITHUB_STEP_SUMMARY - echo "- βœ… Passed: $passed" >> $GITHUB_STEP_SUMMARY - echo "- ❌ Failed: $failed" >> $GITHUB_STEP_SUMMARY - echo "- ⏭️ Not Run: $not_run" >> $GITHUB_STEP_SUMMARY - echo "- πŸ“„ [Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY + if [ "$passed" -gt 0 ]; then + echo "- βœ… Passed: $passed" >> $GITHUB_STEP_SUMMARY + fi + if [ "$failed" -gt 0 ]; then + echo "- ❌ Failed: $failed" >> $GITHUB_STEP_SUMMARY + fi + if [ "$not_run" -gt 0 ]; then + echo "- ⏭️ Not Run: $not_run" >> $GITHUB_STEP_SUMMARY + fi + echo "Pass Rate: $pass_rate" >> $GITHUB_STEP_SUMMARY + echo "Execution Rate: $execution_rate" >> $GITHUB_STEP_SUMMARY + echo "πŸ“„ [Full Report](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY echo "stopping rtsp servers" cd ${{github.workspace}} cd edge-ai-libraries-repo/microservices/dlstreamer-pipeline-server/tests/scripts/utils/ cp -r cvlc_* /tmp/test_results/ ./stream_rtsp.sh stop + - name: Upload Scan artifact to Github - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: Sanity_Reports + name: DLSPS_Sanity-tests-reports path: /tmp/test_results/* + - name: Clean up if: always() run: | diff --git a/.github/workflows/trivy-config-mode.yaml b/.github/workflows/trivy-config-mode.yaml index 723e18d07c..71f67a9178 100644 --- a/.github/workflows/trivy-config-mode.yaml +++ b/.github/workflows/trivy-config-mode.yaml @@ -104,12 +104,12 @@ jobs: --format \"${trivy_report_format}\" \ --output \"${output_report_path}\" \ \"${dockerfile_path}\"" - + # Add Trivy config path if provided if [ -n "$trivy_config_path" ]; then trivy_cmd="$trivy_cmd --config \"$trivy_config_path\"" fi - + eval $trivy_cmd echo "πŸ“„ Report preview:" @@ -118,15 +118,30 @@ jobs: - name: Check Trivy scan failures env: output_report_path: ${{ inputs.output-report-path }} + trivy_report_format: ${{ inputs.trivy-report-format }} run: | - FAILURE_COUNT=$(jq '[.Results[].MisconfSummary.Failures] | add' "${output_report_path}") - echo "Failures found: $FAILURE_COUNT" + if [ "${trivy_report_format}" = "json" ]; then + echo "πŸ“Š Parsing JSON report..." + + # Check if file contains valid JSON + if ! jq empty "${output_report_path}" 2>/dev/null; then + echo "⚠️ Warning: Report is not valid JSON. Skipping failure count check." + exit 0 + fi + + # Extract failure count, handle null/missing fields + FAILURE_COUNT=$(jq '[.Results[]? | .MisconfSummary?.Failures? // 0] | add // 0' "${output_report_path}") + echo "Failures found: $FAILURE_COUNT" - if [ "$FAILURE_COUNT" -gt 0 ]; then - echo "❌ Found $FAILURE_COUNT failures! Failing the job." - exit 1 + if [ "$FAILURE_COUNT" -gt 0 ]; then + echo "❌ Found $FAILURE_COUNT failures! Failing the job." + exit 1 + else + echo "βœ… No failures found. Passing." + fi else - echo "βœ… No failures found. Passing." + echo "ℹ️ Non-JSON format (${trivy_report_format}) detected. Skipping automated failure count." + echo "Please review the report artifact manually." fi - name: Upload Trivy report as artifact diff --git a/libraries/dl-streamer/tests/scripts/installation-on-host-entrypoint.sh b/libraries/dl-streamer/tests/scripts/installation-on-host-entrypoint.sh index 81ae02ae9c..efbc3e94a9 100755 --- a/libraries/dl-streamer/tests/scripts/installation-on-host-entrypoint.sh +++ b/libraries/dl-streamer/tests/scripts/installation-on-host-entrypoint.sh @@ -91,8 +91,8 @@ for file in /usr/share/keyrings/intel-graphics*; do fi done -chmod +x $PREREQUISITES_SCRIPT_PATH/DLS_install_prerequisites.sh -$PREREQUISITES_SCRIPT_PATH/DLS_install_prerequisites.sh --reinstall-npu-driver=no +chmod +x "$PREREQUISITES_SCRIPT_PATH"/DLS_install_prerequisites.sh +"$PREREQUISITES_SCRIPT_PATH"/DLS_install_prerequisites.sh --reinstall-npu-driver=no # Configure repositories before installation echo_color "Starting to configure OpenVINOβ„’ repository access before DL Streamer installation" "blue" diff --git a/microservices/dlstreamer-pipeline-server/tests/requirements.txt b/microservices/dlstreamer-pipeline-server/tests/requirements.txt index b6f8f4ccb2..2182f544ea 100644 --- a/microservices/dlstreamer-pipeline-server/tests/requirements.txt +++ b/microservices/dlstreamer-pipeline-server/tests/requirements.txt @@ -1,5 +1,6 @@ -coverage==7.2.1 -pytest==8.3.4 -pytest-cov==4.0.0 -pytest-mock==3.10.0 -pytest-asyncio==0.25.2 \ No newline at end of file +pytest>=8.3 +pluggy>=1.5 +coverage>=7.6 +pytest-cov>=5.0 +pytest-mock>=3.14 +pytest-asyncio>=0.25.0 diff --git a/microservices/dlstreamer-pipeline-server/tests/test_server_pipeline_manager.py b/microservices/dlstreamer-pipeline-server/tests/test_server_pipeline_manager.py index 69f37967c8..1f5cba30f4 100644 --- a/microservices/dlstreamer-pipeline-server/tests/test_server_pipeline_manager.py +++ b/microservices/dlstreamer-pipeline-server/tests/test_server_pipeline_manager.py @@ -33,7 +33,7 @@ def test_init_pipeline_manager(self,pipeline_manager): assert pipeline_manager.max_running_pipelines == 5 assert pipeline_manager.model_manager is not None assert pipeline_manager.pipeline_dir == "user_pipeline" - + def test_init_failed_load(self,mocker): mocker.patch.object(PipelineManager, '_load_pipelines', return_value=False) with pytest.raises(Exception, match="Error Initializing Pipelines"): @@ -43,7 +43,7 @@ def test_import_pipeline_types(self, pipeline_manager, mocker): mocker.patch('src.server.gstreamer_pipeline.GStreamerPipeline', return_value=MagicMock()) pipeline_types = pipeline_manager._import_pipeline_types() assert "GStreamer" in pipeline_types - mocker.patch.dict('sys.modules',{'src.server.gstreamer_pipeline.GStreamerPipeline':None}, clear=True) + mocker.patch.dict('sys.modules',{'src.server.gstreamer_pipeline':None}) pipeline_types = pipeline_manager._import_pipeline_types() assert {} == pipeline_types @@ -88,7 +88,7 @@ def test_get_loaded_pipelines(self, pipeline_manager, mocker): mock_get_pipeline_params = mocker.patch('src.server.pipeline_manager.PipelineManager.get_pipeline_parameters', return_value = {'name':'pipeline1', 'version': 'v1', "type": "GStreamer"}) pipeline_manager.pipelines = {'pipeline1': {'v1': {'type': 'GStreamer', 'description': 'Test Pipeline'}}} loaded_pipelines = pipeline_manager.get_loaded_pipelines() - assert len(loaded_pipelines) == 1 + assert len(loaded_pipelines) == 1 assert loaded_pipelines[0]["name"] == "pipeline1" assert loaded_pipelines[0]["version"] == "v1" assert loaded_pipelines[0]["type"] == "GStreamer" @@ -158,7 +158,7 @@ def test_stop_instance(self, pipeline_manager): assert pipeline_manager.pipeline_queue == deque(["instance2"]) pipeline_manager.instance_exists.assert_called_once_with("instance1",None,None) result = pipeline_manager.stop_instance('instance3') - assert result + assert result pipeline_manager.instance_exists = MagicMock(return_value=False) result = pipeline_manager.stop_instance('instance2') assert result is None @@ -218,7 +218,7 @@ def test_start_pipeline_manager(self,pipeline_manager,mocker): assert pipeline_manager.running_pipelines == 1 pipeline_manager._get_next_pipeline_identifier.assert_called_once() mock_instance.start.assert_called_once() - + def test_validate_config(self,pipeline_manager,mocker): mock_set_defaults = mocker.patch.object(pipeline_manager,'set_defaults') pipeline_manager.model_manager.model_manager = {"models":"model1"} @@ -405,4 +405,3 @@ def test_load_pipelines_delete(self, pipeline_manager_for_load_pipelines, mocker success = pipeline_manager_for_load_pipelines._load_pipelines() assert success is False assert pipeline_manager_for_load_pipelines.pipelines == {} - \ No newline at end of file