Merge branch 'erlang:master' into data_publisher

Author: zzydxm

.github/scripts/build-macos.sh

@@ -20,6 +20,8 @@
 ##
 ## %CopyrightEnd%
 
+set -e
+
 export MAKEFLAGS="-j$(getconf _NPROCESSORS_ONLN)"
 export ERL_TOP="$(pwd)"
 export ERLC_USE_SERVER=true

.github/scripts/create-openvex-pr.sh

@@ -4,7 +4,7 @@
 ##
 ## SPDX-License-Identifier: Apache-2.0
 ##
-## Copyright Ericsson AB 2026. All Rights Reserved.
+## Copyright Ericsson AB 2026-2025. All Rights Reserved.
 ##
 ## Licensed under the Apache License, Version 2.0 (the "License");
 ## you may not use this file except in compliance with the License.

.github/scripts/ort-scanner.es

@@ -186,14 +186,15 @@ sha1sums(GitRepo) ->
     %% Check which files on disk do not have the same sha1 as the files in the original scan
     FilesToRestore = check_files_to_restore(Input, Cache),
 
+    %% Restores from cache only the files with identical SHA1
     ScanResults = restore_from_cache(FilesToRestore,
                                      InputProvenance,
                                      Cache),
+    ScannersName = [maps:get(~"name", maps:get(~"scanner", R)) || R <- ScanResults],
 
     InputScanner = maps:get(~"scanner", Input),
     InputScanners = maps:get(~"scanners", InputScanner),
-    Scanners = [maps:get(~"name", maps:get(~"scanner", R)) || R <- ScanResults],
-    NewScanners = #{ K => Scanners || K := _ <- InputScanners},
+    NewScanners = #{ K => ScannersName || K := _ <- InputScanners},
 
     CacheConfig = maps:get(~"config", maps:get(~"scanner", Cache)),
 
@@ -428,8 +429,51 @@ replace_mappings(SPDX, Map) ->
     %% other side of NOASSERTION.
     replace_mappings(SPDX, maps:to_list(Map) ++ [{"AND NOASSERTION",""},{"NOASSERTION AND",""}]).
 
+%% deduplicates scan results as per oss-review-toolkit scan-result.json format
 deduplicate(ScanResults) ->
-    deduplicate(ScanResults, []).
+    UniqueScanResults = deduplicate(ScanResults, []),
+
+    %% some results have the exact same provenance and cater to different
+    %% files, so we merge those scan results. this is needed because ORT
+    %% cannot distinguish which of two scan results is the valid one,
+    %% given that they have the same provenance and scanning options.
+    merge_identical_scan_results(UniqueScanResults).
+
+%% merges scan results whose provenance and scanner configuration options
+%% (and scanner versions) are identical. from `deduplicate/2` we are
+%% guaranteed to get scan results with different scanned files
+merge_identical_scan_results(ScanResults) ->
+    Result = lists:foldl(
+               fun (#{~"provenance" := Provenance,
+                      ~"scanner" := Scanner,
+                      ~"summary" := Summary}, Acc) ->
+                       case maps:get({Provenance, Scanner}, Acc, false) of
+                           false ->
+                               Acc#{{Provenance, Scanner} => Summary};
+                           SummaryAcc ->
+                               %% merge results where provenance and scanning options
+                               %% are identical. otherwise ORT (oss-review-toolkit) cannot
+                               %% know which one to choose from two identical configurations.
+                               OldCp = maps:get(~"copyrights", SummaryAcc),
+                               NewCp = maps:get(~"copyrights", Summary),
+
+                               OldLs = maps:get(~"licenses", SummaryAcc),
+                               NewLs = maps:get(~"licenses", Summary),
+
+                               Acc#{{Provenance, Scanner} :=
+                                        Summary#{~"copyrights" := OldCp ++ NewCp,
+                                                 ~"licenses" := OldLs ++ NewLs}}
+                       end
+               end, #{}, ScanResults),
+    maps:fold(fun ({Provenance, Scanner}, V, Acc) ->
+                      [#{~"summary" => V,
+                         ~"provenance" => Provenance,
+                         ~"scanner" => Scanner} | Acc]
+              end, [], Result).
+
+
+%% deduplicates licenses if already found in other scanners.
+%% output: scanners have unique files in their copyrights and licenses
 deduplicate([H | T], Found) ->
     #{ ~"summary" := #{ ~"copyrights" := CS, ~"licenses" := LS} = Summary} = H,
     FilteredCS = [C || C = #{ ~"location" := #{ ~"path" := Path }} <- CS,

.github/scripts/otp-compliance.es

@@ -83,7 +83,7 @@
 -define(spdx_download_location, ~"https://github.com/erlang/otp/releases").
 -define(spdx_homepage, ~"https://www.erlang.org").
 -define(spdx_purl_meta_data, ~"?vcs_url=git+https://github.com/erlang/otp.git").
--define(spdx_version, ~"SPDX-2.2").
+-define(spdx_version, ~"SPDX-2.3").
 -define(otp_version, 'OTP_VERSION'). % file name of the OTP version
 -define(spdx_project_purl, #{ ~"comment" => ~"",
                               ~"referenceCategory" => ~"PACKAGE-MANAGER",
@@ -551,7 +551,8 @@ sbom_fixing_functions(ScanResults) ->
      {fun fix_project_package_version/2, 'OTP_VERSION'},
      {fun fix_has_extracted_license_info/2, extracted_license_info()},
      {fun fix_project_purl/2, ?spdx_project_purl},
-     {fun fix_beam_licenses/2, {Licenses, Copyrights}} ].
+     {fun fix_beam_licenses/2, {Licenses, Copyrights}}
+    ].
 
 fix_project_name(ProjectName, #{ ~"documentDescribes" := [ ProjectName0 ],
                                  ~"packages" := Packages}=Sbom) ->
@@ -1172,7 +1173,7 @@ create_spdx_package(Pkg) ->
     Supplier = Pkg#spdx_package.'supplier',
     Purl1 = case Pkg#spdx_package.'purl' of
                false -> [];
-               _ -> [Pkg#spdx_package.'purl']
+               _ -> Pkg#spdx_package.'purl'
            end,
     #{ ~"SPDXID" => SPDXID,
        ~"versionInfo" => VersionInfo,
@@ -1888,7 +1889,8 @@ create_spdx_package_record(PackageName, Vsn, Description, SpdxPackageFiles,
     VerificationCodeValue = generate_verification_code_value(SpdxPackageFiles),
     Purl1 = case Purl of
                 false -> false;
-                true -> create_externalRef_purl(Description, otp_purl(PackageName, Vsn))
+                true -> [create_externalRef_purl(Description, otp_purl(PackageName, Vsn)),
+                         fix_openvex_reference()]
             end,
     #spdx_package {
        'SPDXID' = SpdxPackageName,
@@ -1911,6 +1913,19 @@ create_spdx_package_record(PackageName, Vsn, Description, SpdxPackageFiles,
       }.
 
 
+fix_openvex_reference() ->
+    OTPMajorVersion = hd(string:split(get_otp_version(), ".")),
+    Reference = openvex_iri(OTPMajorVersion),
+    #{
+     ~"referenceCategory" => ~"SECURITY",
+     ~"referenceLocator" => Reference,
+     ~"referenceType" => ~"advisory"
+    }.
+
+%% Branch = ~"28" or similar. just the current version number.
+openvex_iri(Branch) when is_binary(Branch) ->
+    <<"https://erlang.org/download/vex/otp-", Branch/binary, ".openvex.json">>.
+
 otp_app_license_mapping(Name) ->
     case Name of
         ~"edoc" -> ~"Apache-2.0 OR LGPL-2.1-or-later";
@@ -2375,16 +2390,23 @@ test_project_purl(#{~"documentDescribes" := [ProjectName], ~"packages" := Packag
     ok.
 
 test_packages_purl(#{~"documentDescribes" := [ProjectName], ~"packages" := Packages}=_Sbom) ->
-    OTPPackages = lists:filter(fun (#{~"SPDXID" := Id, ~"name" := Name}) -> ProjectName =/= Id andalso lists:member(Name, minimum_otp_apps()) end, Packages),
-    true = lists:all(fun (#{~"name" := Name, ~"versionInfo" := Version, ~"externalRefs" := [#{~"referenceLocator":= RefLoc}=Ref]}) ->
+    OTPPackages = lists:filter(fun (#{~"SPDXID" := Id, ~"name" := Name}) ->
+                                       ProjectName =/= Id andalso lists:member(Name, minimum_otp_apps())
+                               end, Packages),
+    true = lists:all(fun (#{~"name" := Name, ~"versionInfo" := Version,
+                            ~"externalRefs" := [#{~"referenceLocator":= RefLoc}=Ref,
+                                                OpenVex]}) ->
                              ExternalRef = create_externalRef_purl(~"", otp_purl(Name, Version)),
                              ExternalRef1 = maps:remove(~"comment", ExternalRef),
                              Ref1 = maps:remove(~"comment", Ref),
 
+                             ExpectedVEX = fix_openvex_reference(),
+
                              %% check expected external ref
                              ExternalRef1 =:= Ref1  andalso
                                  %% check metadata is included in purl
-                                 nomatch =/= string:find(RefLoc, ?spdx_purl_meta_data)
+                                 nomatch =/= string:find(RefLoc, ?spdx_purl_meta_data) andalso
+                                 ExpectedVEX == OpenVex
                      end, OTPPackages),
     ok.
 
@@ -3202,12 +3224,13 @@ fetch_app_from_table(OTPVersion, App0) ->
 convert_range(Version) ->
     string:split(Version, ".", all).
 
-
+%% Branch = "otp-28"
 init_openvex_file(Branch) ->
     Ts = calendar:system_time_to_rfc3339(erlang:system_time(microsecond), [{unit, microsecond}]),
+    [~"otp", Version] = string:split(Branch, ~"-"),
     #{
       ~"@context"   => ~"https://openvex.dev/ns/v0.2.0",
-      ~"@id"        => <<"https://openvex.dev/docs/public/otp/vex-", Branch/binary>>,
+      ~"@id"        => openvex_iri(Version),
       ~"author"     => ~"vexctl",
       ~"timestamp"  => erlang:list_to_binary(Ts),
       ~"version"    => 1,

.github/workflows/main.yaml

@@ -59,9 +59,37 @@ permissions:
 
 jobs:
 
+  check-beam:
+    name: Forbid non-maintainers from committing BEAM files
+    runs-on: ubuntu-latest
+    if: github.repository == 'erlang/otp' && github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # ratchet:actions/checkout@v4.2.2
+        with:
+          fetch-depth: 0
+
+      - name: Detect modified BEAM files
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+        run: |
+          PERMISSION=$(gh api \
+                       -H "Accept: application/vnd.github+json" \
+                       -H "X-GitHub-Api-Version: 2022-11-28" \
+                       /repos/{owner}/{repo}/collaborators/$PR_AUTHOR/permission --jq '.role_name')
+
+          MODIFIED_BEAM_FILES=$(git diff --name-only ${{github.event.pull_request.base.sha}} \
+                                                     ${{ github.event.pull_request.head.sha }} | grep '\.beam$' || true)
+          if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" && "$PERMISSION" != "Security Master" && -n "$MODIFIED_BEAM_FILES" ]]; then
+             echo "::error::Workflow failed: Only maintainers can make modifications to '*.beam' files:"
+             echo "$MODIFIED_BEAM_FILES"
+             exit 1
+          fi
+
   pack:
     name: Build Erlang/OTP (64-bit)
     runs-on: ubuntu-latest
+    needs: check-beam
     if: github.repository == 'erlang/otp' || github.event_name != 'scheduled'
     outputs:
         changes: ${{ steps.changes.outputs.changes }}
@@ -194,7 +222,7 @@ jobs:
     needs: pack
     if: needs.pack.outputs.build-c-code == 'true'
     env:
-      WXWIDGETS_VERSION: 3.2.8.1
+      WXWIDGETS_VERSION: 3.2.9
       MACOS_VERSION: 15
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # ratchet:actions/checkout@v4.2.2
@@ -206,7 +234,7 @@ jobs:
 
       - name: Cache wxWidgets
         id: wxwidgets-cache
-        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # ratchet:actions/cache@v4.2.4
         with:
           path: wxWidgets
           key: wxWidgets-${{ env.WXWIDGETS_VERSION }}-${{ runner.os }}-${{ hashFiles('.github/scripts/build-macos-wxwidgets.sh') }}-${{ env.MACOS_VERSION }}
@@ -279,7 +307,7 @@ jobs:
     env:
       WXWIDGETS_VERSION: 3.2.6
     name: Build Erlang/OTP (Windows)
-    runs-on: windows-2022
+    runs-on: windows-2025
     needs: pack
     if: needs.pack.outputs.build-c-code == 'true'
     steps:
@@ -290,6 +318,11 @@ jobs:
       - name: Install WSL dependencies
         run: apt update && apt install -y make autoconf unzip
 
+      - name: Install NSIS
+        shell: cmd
+        run: |
+          choco install nsis --version 3.11.0
+
       - name: Install openssl
         shell: cmd
         run: |
@@ -349,6 +382,7 @@ jobs:
           cp -R wxWidgets /mnt/c/opt/local64/pgm/wxWidgets-${{ env.WXWIDGETS_VERSION }}
           tar -xzf ./otp_src.tar.gz
           cd otp
+          export PATH="/mnt/c/Program Files (x86)/NSIS/bin":$PATH
           export ERL_TOP=`pwd`
           export MAKEFLAGS=-j$(($(nproc) + 2))
           export ERLC_USE_SERVER=true
@@ -796,7 +830,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: pack
     env:
-        ORT_VERSION: 58.0.1
+        ORT_VERSION: 72.0.0
         SCAN_RESULT_CACHE_PATH: .ort/scan-result.json
 
     steps:
@@ -843,6 +877,7 @@ jobs:
       - name: Restore ORT Scanner cache
         uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # ratchet:actions/cache/restore@v4
         id: ort-cache
+        if: ${{ env.FULL_BUILD_AND_CHECK == 'false' }}
         with:
           path: ${{ env.SCAN_RESULT_CACHE_PATH }}
           key: ort-scan-result-${{ env.ORT_VERSION }}-${{ github.ref_name }}-${{ github.sha }}
@@ -858,7 +893,7 @@ jobs:
           FROM otp
           RUN echo 'export PATH="\$HOME/.local/bin:\$PATH"' >> /home/otptest/.profile
           RUN sudo apt-get install -y libicu-dev pip && \
-              pip install click==8.1.7 scancode-toolkit==${SCANCODE_VERSION} reuse && \
+              pip install click==8.3.1 scancode-toolkit==${SCANCODE_VERSION} reuse && \
               pip install -U ntia-conformance-checker
           EOF
 
@@ -886,7 +921,7 @@ jobs:
             sw-version: ${{ env.OTP_SBOM_VERSION }}
 
       - name: Overwrite scan results using reuse
-        run: |          
+        run: |
           cp $HOME/.ort/ort-results/scan-result.json $HOME/.ort/ort-results/scan-result.scanned.json
           docker run -v $PWD/:/github -v $HOME:$HOME otp \
                "/github/.github/scripts/ort-scanner.es scan -t reuse -s overwrite \
@@ -1042,7 +1077,7 @@ jobs:
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # ratchet:actions/download-artifact@v4.3.0
         with:
           name: otp_doc_man
-    
+
       ## We add the correct version name into the file names
       ## and create the hash files for all assets
       - name: Create pre-build and doc archives
@@ -1056,7 +1091,7 @@ jobs:
           FILES=$(ls {*.tar.gz,*.txt})
           md5sum $FILES > MD5.txt
           sha256sum $FILES > SHA256.txt
-    
+
       - name: Download SBoM
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # ratchet:actions/download-artifact@v4.3.0
         with:

.gitignore

@@ -349,5 +349,7 @@ JAVADOC-GENERATED
 
 /lib/xmerl/src/xmerl_sax_parser_*.erl
 /lib/xmerl/src/xmerl_b64Bin.erl
+/lib/xmerl/src/xmerl_xsd_re_parse.erl
 /lib/xmerl/src/xmerl_xpath_parse.erl
 /lib/xmerl/test/xmerl_test.erl
+

.ort/config/config.yml

@@ -36,3 +36,9 @@ ort:
     enabledPackageManagers: [Unmanaged]
     # A flag to control whether excluded scopes and paths should be skipped during the analysis.
     skipExcluded: true
+
+  reporter:
+    reporters:
+      SpdxDocument:
+        options:
+          spdxVersion: "SPDX-2.3"

Makefile.in

@@ -994,3 +994,7 @@ APPS_TEST=$(patsubst %, %_test,$(APPS))
 $(NO_DIALYZER_APPS): $$(patsubst %,%_build,$$@)
 $(APPS_TEST): $$(patsubst %_test,%_build,$$@)
 	ERL_TOP=$(ERL_TOP) TYPE=$(TYPE) $(MAKE) -C lib/$(patsubst %_test,%,$@) test
+
+.PHONY: clean_docs
+clean_docs:
+	rm -rf $(ERL_TOP)/doc $(ERL_TOP)/make/otp_doc_built