add ignore of already open alerts

download gh alerts for otp-compliance to download those existing issues
and ignore them when the PR is scan for vulnerability issues. those
issues should still be sent to GH because otherwise GH considers them
fixed. in the Alerts section from GH, we can mark them as fixed.

Author: kikofernandez

.github/scripts/otp-compliance.es

@@ -231,7 +231,7 @@ cli() ->
 
                                      > .github/scripts/otp-compliance.es sbom osv-scan
                                      """,
-                                 arguments => [ versions_file(), sarif_option(), fail_option() ],
+                                 arguments => [ versions_file(), sarif_option(), fail_option(), gh_alerts() ],
                                  handler => fun osv_scan/1}
                          }},
              "explore" =>
@@ -351,6 +351,11 @@ fail_option() ->
 %% useful for pull requests since we do not want to
 %% add Github Security per found CVE on each PR.
 
+gh_alerts() ->
+    #{name => gh_alerts,
+      type => binary,
+      default => ~"undefined",
+      long => "-gh_alerts"}.
 
 ntia_checker() ->
     #{name => ntia_checker,
@@ -1329,7 +1334,10 @@ generate_vendor_purl(Package) ->
             [create_externalRef_purl(Description, <<Purl/binary, "@", Vsn/binary>>)]
     end.
 
-osv_scan(#{version := Version, sarif := Sarif, fail_if_cve := FailIfCVEFound}) ->
+osv_scan(#{version := Version,
+           sarif := Sarif,
+           fail_if_cve := FailIfCVEFound,
+           gh_alerts := Alerts0}) ->
     application:ensure_all_started([ssl, inets]),
     OSVQuery = vendor_by_version(Version),
 
@@ -1360,8 +1368,24 @@ osv_scan(#{version := Version, sarif := Sarif, fail_if_cve := FailIfCVEFound}) -
             {error, Error} ->
                 {error, [URI, Error]}
         end,
-    Vulns1 = ignore_vex_cves(Vulns),
-    ok = generate_sarif(Version, Sarif, Vulns1),
+    Alerts = case Alerts0 of
+                 ~"undefined" ->
+                     [];
+                 _ ->
+                     decode(Alerts0)
+             end,
+    Vulns1 = ignore_vex_cves(Version, Vulns, Alerts),
+
+    %% Vulnerabilities already open that do not needto be reported again
+    %% in text or make the build fail, but the SARIF file should continue
+    %% reporting until we mark as fixed.
+    io:format("Exiting known vulnerabilities already open:~n~s~n~n",
+              [format_vulnerabilities(Vulns -- Vulns1)]),
+
+    %% sarif should still contain issues with CVEs
+    ok = generate_sarif(Version, Sarif, Vulns),
+
+    %% vulnerability reporting can fail if new issues appear
     FormattedVulns = format_vulnerabilities(Vulns1),
     case FailIfCVEFound of
         false ->
@@ -1425,11 +1449,11 @@ generate_sarif(Branch, Vulns) ->
                              ],
                          ~"partialFingerprints" =>
                              #{ Branch => calculate_fingerprint(Branch, Dependency, Version, CVE)}
-                        } || {Dependency, Version, CVEs} <- Vulns, CVE <- CVEs],
+                        } || {{Dependency, Version}, CVEs} <- Vulns, CVE <- CVEs],
                  ~"artifacts" =>
                      [ #{ ~"location" => #{ ~"uri" => Dependency},
                           ~"length" => -1
-                        } || {Dependency, _, _} <- Vulns]
+                        } || {{Dependency, _}, _} <- Vulns]
                 }]
        }.
 
@@ -1441,22 +1465,47 @@ calculate_fingerprint(Branch, Dependency, Version, CVE) ->
     Bin = crypto:hash(sha, <<Branch/binary, Dependency/binary, Version/binary, CVE/binary>>),
     binary:encode_hex(Bin).
 
-%% TODO: fix by reading VEX files from erlang/vex or repo containing VEX files
-ignore_vex_cves(Vulns) ->
-    lists:foldl(fun ({{~"github.com/wxWidgets/wxWidgets", _Version}, _CVEs}, Acc) ->
+ignore_vex_cves(Branch, Vulns, Alerts) ->
+    Vulns1 = ignore_known_false_positives(Vulns),
+    ignore_dismiss_alerts(Branch, Alerts, Vulns1).
+
+ignore_dismiss_alerts(Branch, Alerts, Vulns) ->
+    FilterBranch = fun (Text) -> string:find(Text, ~"Dependency", trailing) end,
+    CVEsTexts =
+        lists:map(fun (#{ ~"most_recent_instance" := #{~"message" := #{ ~"text":= Text }}}) ->
+                          FilterBranch(Text)
+                  end, Alerts),
+
+    lists:foldl(
+      fun ({{Name, Version}, CVEs}, Acc) ->
+              L = lists:filter(
+                    fun(CVE) ->
+                            T = FilterBranch(error_to_text(Branch, Name, Version, CVE)),
+                            not lists:member(T, CVEsTexts)
+                    end, CVEs),
+              case L of
+                  [] ->
+                      Acc;
+                  _ ->
+                      [{{Name, Version}, L} | Acc]
+              end
+      end, [], Vulns).
+
+ignore_known_false_positives(Vulns) ->
+    lists:foldl(fun ({~"github.com/wxWidgets/wxWidgets", _CVEs}, Acc) ->
                         %% OTP cannot be vulnerable to wxwidgets because
                         %% we only take documentation.
                         Acc;
                     ({{Name, Version}, CVEs}, Acc) ->
                         case maps:get(Name, non_vulnerable_cves(), not_found) of
                             not_found ->
-                                [{Name, Version, CVEs} | Acc];
+                                [{{Name, Version}, CVEs} | Acc];
                             NonCVEs ->
                                 case CVEs -- NonCVEs of
                                     [] ->
                                         Acc;
                                     Vs ->
-                                        [{Name, Version, Vs} | Acc]
+                                        [{{Name, Version}, Vs} | Acc]
                                 end
                         end
                 end, [], Vulns).
@@ -1473,12 +1522,12 @@ non_vulnerable_cves() ->
 format_vulnerabilities({error, ErrorContext}) ->
     {error, ErrorContext};
 format_vulnerabilities(ExistingVulnerabilities) when is_list(ExistingVulnerabilities) ->
-    lists:map(fun ({N, _, Ids}) ->
+    lists:map(fun ({{N, _}, Ids}) ->
                       io_lib:format("- ~s: ~s~n", [N, lists:join(",", Ids)])
               end, ExistingVulnerabilities).
 
 report_vulnerabilities([]) ->
-    io:format("[OSV] No vulnerabilities found.~n");
+    io:format("[OSV] No new vulnerabilities reported.~n");
 report_vulnerabilities({error, [URI, Error]}) ->
     fail("[OSV] POST request to ~p errors: ~p", [URI, Error]);
 report_vulnerabilities(FormatVulns) ->

.github/workflows/main.yaml

@@ -426,8 +426,21 @@ jobs:
                 docker run otp "erl ${OPTION} -noshell -s init stop"
             done
 
+  modified-vendor-files:
+    name: Check if vendor files changed
+    runs-on: ubuntu-latest
+    outputs:
+      vendor-files: ${{ steps.vendor-files.outputs.MODIFIED_FILES != '0' }}
+    steps:
+      - name: Get modified vendor files
+        id: vendor-files
+        run: |
+          echo "MODIFIED_FILES=$(git diff --name-only '${{ github.base_ref }}' 'HEAD' | grep 'vendor\.info$' | wc -l || 1)"
+
   # this is a call to a workflow_call
   pr-vendor-vulnerability-analysis:
+    needs: modified-vendor-files
+    if: ${{ needs.modified-vendor-files.outputs.vendor-files }}
     permissions:
       security-events: write
     name: Vendor Vulnerability Scanning

.github/workflows/osv-scanner-scheduled.yml

@@ -79,5 +79,5 @@ jobs:
             --method POST \
             -H "Accept: application/vnd.github+json" \
             -H "X-GitHub-Api-Version: 2022-11-28" \
-            /repos/${{ env.REPO }}/actions/workflows/reusable-vendor-vulnerability-scanner.yml/dispatches \
+            /repos/${{ github.repository }}/actions/workflows/reusable-vendor-vulnerability-scanner.yml/dispatches \
             -f "ref=master" -f "inputs[version]=${{ matrix.type }}" -f "inputs[sarif]=true" -f "inputs[fail_if_cve]=false"

.github/workflows/reusable-vendor-vulnerability-scanner.yml

@@ -50,7 +50,10 @@ on:
         default: false
         type: boolean
       fail_if_cve:
-        description: 'The build fails if a CVE is found. This is ok to activate in PRs, but does not make sense in scheduled analysis since CVEs will be reported in Github Security via SARIF file upload.'
+        # The build fails if a CVE is found. This is ok to activate in PRs, but
+        # does not make sense in scheduled analysis since CVEs will be reported
+        # in Github Security
+        description: 'Fail if CVE is found'
         required: true
         default: false
         type: boolean
@@ -67,15 +70,16 @@ on:
         default: false
         type: boolean
       fail_if_cve:
-        description: 'The build fails if a CVE is found. This is ok to activate in PRs, but does not make sense in scheduled analysis since CVEs will be reported in Github Security.'
+        # The build fails if a CVE is found. This is ok to activate in PRs, but
+        # does not make sense in scheduled analysis since CVEs will be reported
+        # in Github Security.
+        description: 'Fail if CVE is found'
         required: true
         default: false
         type: boolean
 
 env:
   VERSION: ${{ inputs.version }}
-  REPO: "erlang/otp"
-  # when testing, change to fork, e.g., "kikofernandez/otp"
 
 jobs:
   analysis-vendor-dependencies:
@@ -86,30 +90,45 @@ jobs:
     # committed into them. thus, a workflow_dispatch or workflow_call would
     # not work, and we would not be able to analyse vendor dependecies there.
     runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     permissions:
       security-events: write
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # ratchet:actions/[email protected]
       - uses: erlef/setup-beam@5304e04ea2b355f03681464e683d92e3b2f18451 # racket:actions/checkout@v1
         with:
           otp-version: '27'
+
+      - name: Download Scanning Alerts
+        run: |
+          gh api -H "Accept: application/vnd.github+json" \
+                 -H "X-GitHub-Api-Version: 2022-11-28" \
+                 /repos/${{ github.repository }}/code-scanning/alerts?tool_name=otp-compliance > "$HOME/gh_alerts.json"
+
       - name: 'Analysis of dependencies in ${{ inputs.version }}'
+        id: analysis
         run: |
-          git clone -b ${{ env.VERSION }} https://github.com/${{ env.REPO }}.git ${{ env.VERSION }}
+          git clone -b ${{ env.VERSION }} https://github.com/${{ github.repository }}.git ${{ env.VERSION }}
           mkdir -p /home/runner/work/otp/otp/${{ env.VERSION }}/.github/scripts/
-          cp /home/runner/work/otp/otp/.github/scripts/otp-compliance.es \
+          curl -LJO https://raw.githubusercontent.com/${{ github.repository }}/refs/heads/master/.github/scripts/otp-compliance.es
+          chmod +x otp-compliance.es
+          cp otp-compliance.es \
              /home/runner/work/otp/otp/${{ env.VERSION }}/.github/scripts/otp-compliance.es
           cd /home/runner/work/otp/otp/${{ env.VERSION }} && \
-             .github/scripts/otp-compliance.es sbom osv-scan --version ${{ inputs.version }} --fail_if_cve ${{ inputs.fail_if_cve }}
+          .github/scripts/otp-compliance.es sbom osv-scan \
+               --version ${{ inputs.version }} \
+               --fail_if_cve ${{ inputs.fail_if_cve }} \
+               --gh_alerts $HOME/gh_alerts.json
 
       - name: "Get SHA"
+        if: ${{ !failure() && steps.analysis.outcome != 'skipped' && inputs.sarif }}
         id: sha
-        if: ${{ !failure() && inputs.sarif }}
         run: |
           echo "sha=$(cd /home/runner/work/otp/otp/${{ env.VERSION }} && git rev-parse HEAD)" >> $GITHUB_OUTPUT
 
       - name: "Upload to code-scanning"
-        if: ${{ !failure() && inputs.sarif }}
+        if: ${{ !failure() && steps.analysis.outcome != 'skipped' && inputs.sarif }}
         env:
           SHA: ${{ steps.sha.outputs.sha }}
         uses: github/codeql-action/upload-sarif@ea9e4e37992a54ee68a9622e985e60c8e8f12d9f # ratchet:github/codeql-action/upload-sarif@v3