otp scan PRs for vulnerabilities

- scan PRs for vendor vulnerabilities.
- the submission of the vendor SBOM should happen only on push events.
- vulnerability scanning of dependencies must happen on a per PR basis,
  and on a per push basis (although Dependatbot should inform us of this).

Author: kikofernandez

.github/scripts/otp-compliance.es

@@ -216,7 +216,18 @@ cli() ->
                                      > .github/scripts/otp-compliance.es sbom vendor --sbom-file otp.spdx.json
                                      """,
                                  arguments => [ sbom_option()],
-                                 handler => fun sbom_vendor/1}
+                                 handler => fun sbom_vendor/1},
+
+                          "osv-scan" =>
+                              #{ help =>
+                                     """
+                                     Performs vulnerability scanning on vendor libraries
+
+                                     Example:
+
+                                     > .github/scripts/otp-compliance.es sbom osv-scan
+                                     """,
+                                 handler => fun osv_scan/1}
                          }},
              "explore" =>
                  #{  help => """
@@ -1205,6 +1216,87 @@ generate_vendor_purl(Package) ->
             [create_externalRef_purl(Description, <<Purl/binary, "@", Vsn/binary>>)]
     end.
 
+osv_scan(_) ->
+    application:ensure_all_started([ssl, inets]),
+
+    VendorSrcFiles = find_vendor_src_files("."),
+    Packages = generate_vendor_info_package(VendorSrcFiles),
+
+    %% Test if this works in a Github Workflow
+    OSVQueryResults = generate_osv_results(Packages),
+    io:format("~s", [json:format(OSVQueryResults)]).
+    %% file:write_file(File, json:format(OSVQueryResults)).
+
+    %% OSVQuery = generate_osv_query(Packages),
+%%     io:format("[OSV] Information sent~n~s~n", [json:format(OSVQuery)]),
+%%     OSV = json:encode(OSVQuery),
+
+%%     Content = {URI, [], Format, OSV},
+%%     Result = httpc:request(post, Content, [], []),
+%%     case Result of
+%%         {ok,{{_, 200,_}, _Headers, Body}} ->
+%%             #{~"results" := OSVResults} = json:decode(erlang:list_to_binary(Body)),
+%%             Vulnerabilities = lists:filter(fun (#{~"vulns" := _Ids}) -> true; (_) -> false end, OSVResults),
+%%             case Vulnerabilities of
+%%                 [] ->
+%%                     io:format("[OSV] No vulnerabilities found.~n");
+%%                 _ ->
+%%                     FormatVulns = format_vulnerabilities(OSVQuery, OSVResults),
+%%                     fail("[OSV] There are existing vulnerabilities:~n~s", [FormatVulns])
+%%             end;
+%%         {error, Error} ->
+%%             fail("[OSV] POST request to ~p errors: ~p", [URI, Error])
+%%     end.
+
+%% format_vulnerabilities(OSVQuery, OSVResults) ->
+%%     NameVulnerabilities = lists:zip(osv_names(OSVQuery), OSVResults),
+%%     ExistingVulnerabilities = lists:filtermap(fun ({Name, #{~"vulns" := Ids}}) ->
+%%                                                       {true, {Name, [Id || #{~"id" := Id} <- Ids]}};
+%%                                                   (_) ->
+%%                                                       false
+%%                                               end, NameVulnerabilities),
+%%     lists:map(fun ({N, Ids}) ->
+%%                       io_lib:format("- ~s: ~s~n", [N, lists:join(",", Ids)])
+%%               end, ExistingVulnerabilities).
+
+%% osv_names(#{~"queries" := Packages}) ->
+%%     lists:map(fun osv_names/1, Packages);
+%% osv_names(#{~"package" := #{~"name" := Name }}) ->
+%%     Name.
+
+%% generate_osv_query(Packages) ->
+%%     #{~"queries" => lists:foldl(fun generate_osv_query/2, [], Packages)}.
+%% generate_osv_query(#{~"versionInfo" := Vsn, ~"ecosystem" := Ecosystem, ~"name" := Name}, Acc) ->
+%%     Package = #{~"package" => #{~"name" => Name, ~"ecosystem" => Ecosystem}, ~"version" => Vsn},
+%%     [Package | Acc];
+%% generate_osv_query(#{~"sha" := SHA, ~"downloadLocation" := Location}, Acc) ->
+%%     case string:prefix(Location, ~"https://") of
+%%         nomatch ->
+%%             Acc;
+%%         URI ->
+%%             Package = #{~"package" => #{~"name" => URI}, ~"commit" => SHA},
+%%             [Package | Acc]
+%%     end;
+%% generate_osv_query(_, Acc) ->
+%%     Acc.
+
+
+generate_osv_results(Packages) ->
+    #{~"results" => [#{~"packages" => lists:foldl(fun generate_osv_results/2, [], Packages)}]}.
+generate_osv_results(#{~"versionInfo" := Vsn, ~"ecosystem" := Ecosystem, ~"name" := Name}, Acc) ->
+    Package = #{~"package" => #{~"name" => Name, ~"ecosystem" => Ecosystem, ~"version" => Vsn}},
+    [Package | Acc];
+generate_osv_results(#{~"sha" := SHA, ~"downloadLocation" := Location}, Acc) ->
+    case string:prefix(Location, ~"https://") of
+        nomatch ->
+            Acc;
+        URI ->
+            Package = #{~"package" => #{~"name" => URI, ~"commit" => SHA}},
+            [Package | Acc]
+    end;
+generate_osv_results(_, Acc) ->
+    Acc.
+
 cleanup_path(<<"./", Path/binary>>) when is_binary(Path) -> Path;
 cleanup_path(Path) when is_binary(Path) -> Path.
 

.github/workflows/main.yaml

@@ -425,6 +425,50 @@ jobs:
                 docker run otp "erl ${OPTION} -noshell -s init stop"
             done
 
+  extract-deps:
+    name: Extract Dependencies
+    runs-on: ubuntu-latest
+    needs:
+      - pack
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # ratchet:actions/[email protected]
+      - uses: ./.github/actions/build-base-image
+        with:
+            BASE_BRANCH: ${{ env.BASE_BRANCH }}
+
+      # check that PRs do not introduce vulnerabilities in vendor dependencies
+      - name: 'Vendor Vulnerability Scanning'
+        run: |
+          docker run -v $PWD/:/github otp \
+            "/github/.github/scripts/otp-compliance.es sbom osv-scan > /github/osv-scanner.json"
+          sudo chown -R `whoami` osv-scanner.json
+
+      # Upload the deps
+      - name: "upload osv-scanner deps"
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # ratchet:actions/[email protected]
+        with:
+          name: converted-OSV-Scanner-deps
+          path: /home/runner/work/otp/otp/osv-scanner.json
+          retention-days: 2
+
+  scan-pr:
+    # run-scheduled-scan triggers this job
+    # PRs and pushes trigger this job
+    needs: extract-deps
+    permissions:
+        # Require writing security events to upload SARIF file to security tab
+        security-events: write
+        # Required to upload SARIF file to CodeQL.
+        # See: https://github.com/github/codeql-action/issues/2117
+        actions: read
+        contents: read
+    uses: "google/osv-scanner-action/.github/workflows/osv-scanner-reusable.yml@6fc714450122bda9d00e4ad5d639ad6a39eedb1f" # ratchet:google/osv-scanner-action/.github/workflows/[email protected]
+    with:
+        download-artifact: converted-OSV-Scanner-deps
+        upload-sarif: ${{ github.repository == 'erlang/otp' }}
+        scan-args: |-
+            --lockfile=osv-scanner:osv-scanner.json
+
   build:
     name: Build Erlang/OTP
     runs-on: ubuntu-latest
@@ -847,10 +891,9 @@ jobs:
             fail-on: ${{ github.ref_type == 'tag' && '' || '' }} # 'violations,issues' }}
             sw-version: ${{ env.OTP_SBOM_VERSION }}
 
-  vendor-analysis:
-    name: Vendor Dependency Analysis
+  vendor-dependency-upload:
+    name: Vendor Dependency Upload
     runs-on: ubuntu-latest
-    if: github.event_name == 'push'
     needs:
         - sbom
         - pack
@@ -878,6 +921,7 @@ jobs:
 
       # allows Dependabot to give us alert of the vendor libraries that use semantic versioning
       - name: Upload SBOM to Github Dependency API
+        if: github.event_name == 'push'
         uses: advanced-security/spdx-dependency-submission-action@5530bab9ee4bbe66420ce8280624036c77f89746 # ratchet:advanced-security/[email protected]
 
   ## If this is an "OTP-*" tag that has been pushed we do some release work

HOWTO/SBOM.md

@@ -183,6 +183,7 @@ This file may be a list of JSON objects. For simplicity, we document the fields
     "licenseDeclared": "Zlib",
     "name": "asmjit",
     "versionInfo": "a465fe71ab3d0e224b2b4bd0fac69ae68ab9239d",
+    "sha": "a465fe71ab3d0e224b2b4bd0fac69ae68ab9239d",
     "path": "./erts/emulator/asmjit",
     "supplier": "Person: Petr Kobalicek",
     "purl": "pkg:github/asmjit/asmjit"
@@ -199,11 +200,15 @@ Fields summary:
         - If you are unsure about the name of the `SPDX-TOP-LEVEL-PACKAGE`, take a look at the source SBOM to identify packages (under key `packages` in the SBOM).
 - `description`: a brief description of what this vendor library does.
 - `copyrightText`: copyright text associated with the top-level package/library/3pp using [SPDX License Identifiers](https://spdx.org/licenses/).
-- `downloadLocation`: URI of the vendor library to download.
+- `downloadLocation`: URI of the vendor library to download. If using Github, use preferably `https//` rather than `git+https//` or similars.
+   This is because the download location is used for vulnerability scanning in `.github/scripts/otp-compliance.es`.
 - `homepage`: homepage of the vendor library.
 - `licenseDeclared`: license as declared by the vendor, following a [SPDX license identifier](https://spdx.org/licenses/).
 - `name`: name of the library.
 - `versionInfo`: version of the library/project/3pp. In case of no version number being available, write the commit sha.
+- `sha`: sha commit for `versionInfo`, they need to be updated together!
+- `ecosystem`: List of valid ecosystems in [OSV Ecosystems](https://ossf.github.io/osv-schema/#defined-ecosystems)
+  where this value is omitted for C/C++ code (e.g., `asmjit`, `pcre2`, `zlib`, `zstd`, etc), and used in `vendor.json` for `jquery`.
 - `path`: path to the vendor library inside Erlang/OTP. This can point to a folder or a list of files.
   - Folder: any file inside the folder is considered part of the vendor library (e.g., asmjit [vendor.info](../erts/emulator/asmjit/vendor.info)).
   - List of files: only the files listed here are part of a vendor library (e.g., erts-config [vendor.info](../erts/autoconf/vendor.info)).

erts/emulator/asmjit/vendor.info

@@ -15,6 +15,7 @@
     "licenseDeclared": "Zlib",
     "name": "asmjit",
     "versionInfo": "a465fe71ab3d0e224b2b4bd0fac69ae68ab9239d",
+    "sha": "a465fe71ab3d0e224b2b4bd0fac69ae68ab9239d",
     "path": "./erts/emulator/asmjit",
     "supplier": "Person: Petr Kobalicek",
     "purl": "pkg:github/asmjit/asmjit"

erts/emulator/openssl/vendor.info

@@ -15,6 +15,7 @@
     "licenseDeclared": "Apache-2.0",
     "name": "openssl",
     "versionInfo": "3.5",
+    "sha": "cdfb0923a66155ce97640fca68ae57b3a2972029",
     "path": "./erts/emulator/openssl",
     "supplier": "Organization: OpenSSL Mission",
     "purl": "pkg:generic/openssl"

erts/emulator/pcre/vendor.info

@@ -10,12 +10,13 @@
     "ID": "erts-pcre",
     "description": "PCRE2 library",
     "copyrightText": "NOASSERTION",
-    "downloadLocation": "git+https://github.com/PCRE2Project/pcre2.git",
+    "downloadLocation": "https://github.com/PCRE2Project/pcre2",
     "homepage": "https://pcre2project.github.io/pcre2/",
     "licenseDeclared": "BSD-3-Clause",
     "name": "pcre2",
-    "versionInfo": "10.44",
+    "versionInfo": "10.45",
     "path": "./erts/emulator/pcre",
+    "sha": "1e09555d6950bfcf83bd98fa597b0c6440d43c9c",
     "supplier": "Person: Philip Hazel",
     "purl": "pkg:generic/pcre2"
   }

erts/emulator/zlib/vendor.info

@@ -10,13 +10,14 @@
     "ID": "erts-zlib",
     "description": "interface of the 'zlib' general purpose compression library",
     "copyrightText": "Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler",
-    "downloadLocation": "https://zlib.net/",
+    "downloadLocation": "https://github.com/madler/zlib",
     "homepage": "https://zlib.net/",
     "licenseDeclared": "Zlib",
     "name": "zlib",
     "versionInfo": "1.3.1",
+    "sha": "1a8db63788c34a50e39e273d39b7e1033208aea2",
     "path": "./erts/emulator/zlib",
     "supplier": "Person: Mark Adler ([email protected])",
-    "purl": "pkg:generic/zlib"
+    "purl": "pkg:github/madler/zlib"
   }
 ]

erts/emulator/zstd/vendor.info

@@ -16,6 +16,7 @@
     "licenseDeclared": "BSD-3-Clause OR GPL-2.0-only",
     "name": "zstd",
     "versionInfo": "v1.5.7",
+    "sha": "f8745da6ff1ad1e7bab384bd1f9d742439278e99",
     "path": "./erts/emulator/zstd",
     "supplier": "Organization: Meta",
     "purl": "pkg:github/facebook/zstd",

lib/common_test/priv/vendor.info

@@ -13,6 +13,7 @@
       "downloadLocation": "https://github.com/jquery/jquery",
       "homepage": "https://jquery.com",
       "licenseDeclared": "MIT",
+      "ecosystem": "npm",
       "name": "jquery",
       "versionInfo": "3.7.1",
       "path": ["./lib/common_test/priv/jquery-latest.js"],
@@ -26,7 +27,8 @@
       "downloadLocation": "https://github.com/Mottie/tablesorter",
       "homepage": "https://github.com/Mottie/tablesorter",
       "licenseDeclared": "BSD-3-Clause OR GPL-2.0-only",
-      "name": "jquery-tablesorter",
+      "ecosystem": "npm",
+      "name": "tablesorter",
       "versionInfo": "2.32",
       "path": ["./lib/common_test/priv/jquery.tablesorter.min.js"],
       "supplier": "Person: Christian Bach",

lib/erl_interface/src/openssl/vendor.info

@@ -15,6 +15,7 @@
     "licenseDeclared": "Apache-2.0",
     "name": "openssl",
     "versionInfo": "3.5",
+    "sha": "636dfadc70ce26f2473870570bfd9ec352806b1d",
     "path": "./lib/erl_interface/src/openssl",
     "supplier": "Organization: OpenSSL Mission",
     "purl": "pkg:generic/openssl"

lib/wx/vendor.info

@@ -14,9 +14,10 @@
     "homepage": "https://github.com/wxWidgets/wxWidgets",
     "licenseDeclared": "LicenseRef-scancode-wxwindows-free-doc-3",
     "name": "wx",
-    "versionInfo": "dc585039bbd426829e3433002023a93f9bedd0c2",
+    "versionInfo": "f2918a9ac823074901ce27de939baa57788beb3d",
+    "sha": "fd0f60daea24e9c62d372d774be9e32ce2b0849d",
     "path": "./lib/wx",
-    "comments": "This only applies to the source code of Erlang files in 'src', and specifically to the documentation embedded in them",
+    "comments": "This only applies to the source code of Erlang files in 'src', and specifically to the documentation embedded in them.",
     "supplier": "NOASSERTION",
     "purl": "pkg:github/wxwidgets/wxwidgets"
   }