Add upgrade script for ryu

.github/scripts/ort-scanner.es

@@ -487,8 +487,8 @@ format_scan_results(ScanResult) ->
         end
      end)).
 
-sort_arrays(JSon) ->
-    sort_arrays(JSon, fun(_) -> true end).
+% sort_arrays(JSon) ->
+%     sort_arrays(JSon, fun(_) -> true end).
 sort_arrays(JSon, KeyFun) ->
     sort_arrays(JSon, KeyFun, true).
 sort_arrays(Map, KeyFun, _Sort) when is_map(Map) ->

.github/scripts/otp-compliance.es

@@ -53,7 +53,7 @@
          test_package_ids/1, test_verificationCode/1, test_supplier_Ericsson/1,
          test_originator_Ericsson/1, test_versionInfo_not_empty/1, test_package_hasFiles/1,
          test_project_purl/1, test_packages_purl/1, test_download_location/1, 
-         test_package_relations/1, test_has_extracted_licenses/1, test_snippets/1,
+         test_package_relations/1, test_has_extracted_licenses/1,
          test_vendor_packages/1]).
 
 -define(default_classified_result, "scan-result-classified.json").
@@ -365,10 +365,7 @@ sbom_otp(#{sbom_file  := SbomFile, write_to_file := Write, input_file := Input})
 
 improve_sbom_with_info(Sbom, ScanResults) ->
     {Licenses, Copyrights} = fetch_license_copyrights(ScanResults),
-    Spdx = generate_spdx_fixes(Sbom, Licenses, Copyrights),
-    generate_snippet_fixes(Spdx, ScanResults).
-    %% Spdx.
-
+    generate_spdx_fixes(Sbom, Licenses, Copyrights).
 
 fetch_license_copyrights(Input) ->
     {path_to_license(Input), path_to_copyright(Input)}.
@@ -379,68 +376,6 @@ generate_spdx_fixes(Input, Licenses, Copyrights) ->
     Spdx = lists:foldl(fun ({Fun, Data}, Acc) -> Fun(Data, Acc) end, Input, FixFuns),
     package_by_app(Spdx).
 
-%% this function has a hard dependency to ScanResults.
-%% ScanResults has license results on a per line found basis.
-%% Spdx result builds using ScanResult and loses this information.
-generate_snippet_fixes(Spdx, ScanResults) ->
-    Licenses = licenses(scan_results(ScanResults)),
-    %% We identify the known copied snippet
-    [Snippet] = lists:filter(fun(#{~"location" := #{~"path" := Path}, ~"license" := License}) ->
-                                    case {License, Path} of
-                                        {~"Apache-2.0 WITH LLVM-exception AND BSL-1.0", ~"erts/emulator/ryu/d2s.c"} ->
-                                            true;
-                                        _ ->
-                                            false
-                                    end
-                            end, Licenses),
-    Snippets = generate_snippet(Spdx, [Snippet]),
-    Spdx#{ ~"snippets" => Snippets }.
-
-%% we are doing the assumption that we only have one known snippet.
-generate_snippet(#{~"files" := Files}=_Spdx, [#{~"location" := #{~"path" := ~"erts/emulator/ryu/d2s.c"=Path}, ~"license" := License}]) ->
-    [#{~"SPDXID" := SpdxId}=SpdxFile] = lists:filter(fun (#{~"fileName" := FileName}) -> FileName == Path end, Files),
-
-    %% read file to find snippet byte range and lines
-    {ok, Content} = file:read_file(Path),
-    #{~"begin_byte" := StartOffsetBytes, ~"end_byte" := EndOffsetBytes,
-      ~"start_line" := StartLine,        ~"end_line" := EndLine} = get_snippet_range(Content),
-
-    [#{ ~"SPDXID" => ~"SPDXRef-Snippet-STL",
-        ~"comment" => ~"""
-                       vendor package.
-                       This is inspired from the MS STL Charconv, under Apache with LLVM exception licence see https://github.com/microsoft/STL/blob/main/LICENSE.txt
-                       The inspiration is at https://github.com/microsoft/STL/blob/e745bad3b1d05b5b19ec652d68abb37865ffa454/stl/inc/xcharconv_ryu.h#L1926
-                       Changes are described in the file.
-                       """,
-        ~"copyrightText" => maps:get(~"copyrightText", SpdxFile),
-        ~"licenseConcluded" => ~"Apache-2.0 WITH LLVM-exception AND BSL-1.0",
-        ~"licenseInfoInSnippets" => split_licenses_in_individual_parts([License]),
-        ~"name" => ~"stl",
-        ~"ranges" => [ #{~"endPointer" => #{ ~"lineNumber" => EndLine, ~"reference" => SpdxId},
-                         ~"startPointer" => #{ ~"lineNumber" => StartLine, ~"reference" => SpdxId}},
-                       #{~"endPointer" => #{ ~"offset" => EndOffsetBytes, ~"reference" => SpdxId},
-                         ~"startPointer" => #{ ~"offset" => StartOffsetBytes, ~"reference" => SpdxId}} ],
-        ~"snippetFromFile" => SpdxId
-      }].
-
-%% assumes that there is only one snippet in the whole file.
--spec get_snippet_range(Content :: binary()) -> map().
-get_snippet_range(Content) ->
-    get_snippet_range(Content, {<<>>, 1}).
-get_snippet_range(<<"\n", Rest/binary>>, {Acc, Lines}) ->
-    get_snippet_range(Rest, {<<"\n", Acc/binary>>, Lines+1});
-get_snippet_range(<<"// SPDX-SnippetBegin", Rest/binary>>, {Content, Line}) ->
-    {BeginContent, StartLine} = {Content, Line},
-    {EndContent, EndLine} = get_snippet_range(Rest, {<<"// SPDX-SnippetBegin", Content/binary>>, Line+1}),
-    #{~"begin_byte" => byte_size(BeginContent),
-      ~"end_byte" => byte_size(EndContent),
-      ~"start_line" => StartLine,
-      ~"end_line" => EndLine};
-get_snippet_range(<<"// SPDX-SnippetEnd", _Rest/binary>>, {Content, Line}) ->
-    {Content, Line};
-get_snippet_range(<<C, Rest/binary>>, {Content, Line}) ->
-    get_snippet_range(Rest, {<<C, Content/binary>>, Line}).
-
 sbom_fixing_functions(Licenses, Copyrights) ->
     [{fun fix_project_name/2, ?spdxref_project_name},
      {fun fix_name/2, ?spdx_project_name},
@@ -580,14 +515,6 @@ fix_beam_licenses(LicensesAndCopyrights,
                               %% follows from otp/lib/stdlib/uc_spec/README-UPDATE.txt
                               files_have_no_license(SPDX#{~"licenseConcluded" := ~"Unicode-3.0 AND Apache-2.0"});
 
-                          #{~"fileName" := <<"erts/emulator/internal_doc/",Filename/binary>>} ->
-                              case binary:split(Filename, ~".md") of
-                                  [_File, _Ext] ->
-                                      SPDX#{~"licenseConcluded" := ~"Apache-2.0"};
-                                  _ ->
-                                      SPDX
-                              end;
-
                           #{~"fileName" := Filename} ->
                               case bootstrap_mappings(Filename) of
                                   {error, not_beam_file} ->
@@ -858,7 +785,7 @@ decode_without_spdx_license(Filename) ->
 
     %% remove comments
     Lines = string:split(Bin, "\n", all),
-    Lines1 = lists:map(fun (Line) -> re:replace(Line, "%.*", "", [global]) end, Lines),
+    Lines1 = lists:map(fun (Line) -> re:replace(Line, "^//.*", "", [global]) end, Lines),
     Bin1 = erlang:iolist_to_binary(Lines1),
 
     json:decode(Bin1).
@@ -1141,8 +1068,7 @@ get_otp_apps_from_table() ->
     end.
 
 find_vendor_src_files(Folder) ->
-    S = os:cmd("find "++ Folder ++ " -name vendor.info"),
-    lists:map(fun erlang:list_to_binary/1, string:split(S, "\n", all)).
+    string:split(string:trim(os:cmd("find "++ Folder ++ " -name vendor.info")), "\n", all).
 
 -spec generate_spdx_mappings(Path :: [binary()]) -> Result when
       Result :: #{AppName :: binary() => {AppPath :: binary(), AppInfo :: app_info()}}.
@@ -1152,31 +1078,18 @@ generate_spdx_mappings(AppSrcPath) ->
                         maps:merge(Acc, DetectedPackages)
                 end, #{}, AppSrcPath).
 
--spec generate_vendor_info_package(VendorSrcPath :: [binary()]) -> map().
-generate_vendor_info_package(VendorSrcPath) ->
-    lists:foldl(fun vendor_info_to_map/2, [], VendorSrcPath).
-
-
 %% Read Path file and generate Json (map) following vendor.info specification
-vendor_info_to_map(<<>>, Acc) ->
-    Acc;
-vendor_info_to_map(Path, Acc) ->
-    case decode_without_spdx_license(Path) of
-        Json when is_list(Json) ->
-            Json ++ Acc;
-        Json when is_map(Json) ->
-            [Json | Acc]
-    end.
+-spec generate_vendor_info_package(VendorSrcPath :: [file:name()]) -> map().
+generate_vendor_info_package(VendorSrcPath) ->
+    lists:flatmap(fun decode_without_spdx_license/1, VendorSrcPath).
 
 -spec generate_spdx_vendor_packages(VendorInfoPackage :: map(), map()) -> map().
 generate_spdx_vendor_packages(VendorInfoPackages, #{~"files" := SpdxFiles}=_SPDX) ->
     lists:map(fun
                   (#{~"ID" := Id, ~"path" := [_ | _]=ExplicitFiles}=Package) when is_list(ExplicitFiles) ->
                       %% Deals with the cases of creating a package out of specific files
                       Paths = lists:map(fun cleanup_path/1, ExplicitFiles),
-                      Package0 = maps:remove(~"purl", Package),
-                      Package1 = maps:remove(~"ID", Package0),
-                      Package2 = maps:remove(~"path", Package1),
+                      Package1 = maps:without([~"purl", ~"ID", ~"path", ~"update"], Package),
 
                       %% place files in SPDX in the corresponding package
                       Files = lists:filter(fun (#{~"fileName" := Filename}) ->
@@ -1190,7 +1103,7 @@ generate_spdx_vendor_packages(VendorInfoPackages, #{~"files" := SpdxFiles}=_SPDX
 
                       PackageVerificationCodeValue = generate_verification_code_value(Files),
                       ExternalRefs = generate_vendor_purl(Package),
-                      Package2#{
+                      Package1#{
                                 ~"SPDXID" => generate_spdxid_name(Id),
                                 ~"filesAnalyzed" => true,
                                 ~"hasFiles" => lists:map(fun (#{~"SPDXID":=Id0}) -> Id0 end, Files),
@@ -1203,9 +1116,7 @@ generate_spdx_vendor_packages(VendorInfoPackages, #{~"files" := SpdxFiles}=_SPDX
                   (#{~"ID" := Id, ~"path" := DirtyPath}=Package) when is_binary(DirtyPath) ->
                       %% Deals with the case of creating a package out of a path
                       Path = cleanup_path(DirtyPath),
-                      Package0 = maps:remove(~"purl", Package),
-                      Package1 = maps:remove(~"ID", Package0),
-                      Package2 = maps:remove(~"path", Package1),
+                      Package1 = maps:without([~"purl", ~"ID", ~"path", ~"update"], Package),
 
                       %% place files in SPDX in the corresponding package
                       Files = lists:filter(fun (#{~"fileName" := Filename}) ->
@@ -1221,7 +1132,7 @@ generate_spdx_vendor_packages(VendorInfoPackages, #{~"files" := SpdxFiles}=_SPDX
 
                       PackageVerificationCodeValue = generate_verification_code_value(Files),
                       ExternalRefs = generate_vendor_purl(Package),
-                      Package2#{
+                      Package1#{
                                 ~"SPDXID" => generate_spdxid_name(Id),
                                 ~"filesAnalyzed" => true,
                                 ~"hasFiles" => lists:map(fun (#{~"SPDXID":=Id0}) -> Id0 end, Files),
@@ -1523,7 +1434,6 @@ package_generator(Sbom) ->
              test_download_location,
              test_package_relations,
              test_has_extracted_licenses,
-             test_snippets,
              test_vendor_packages],
     true = ?CALL_TEST_FUNCTIONS(Tests, Sbom),
     ok.
@@ -1587,7 +1497,7 @@ root_vendor_packages() ->
 minimum_vendor_packages() ->
     %% self-contained
     root_vendor_packages() ++
-        [~"tcl", ~"json-test-suite", ~"openssl", ~"Autoconf", ~"wx", ~"jquery", ~"jquery-tablesorter"].
+        [~"tcl", ~"ryu_to_chars", ~"json-test-suite", ~"openssl", ~"Autoconf", ~"wx", ~"jquery", ~"jquery-tablesorter"].
 
 test_copyright_not_empty(#{~"packages" := Packages}) ->
     true = lists:all(fun (#{~"copyrightText" := Copyright}) -> Copyright =/= ~"" end, Packages),
@@ -1602,7 +1512,7 @@ test_hasFiles_not_empty(#{~"packages" := Packages}) ->
         true = lists:all(fun (#{~"hasFiles" := Files}) -> length(Files) > 0 end, Packages)
     catch
         _:_:_ ->
-            lists:map(fun (#{~"hasFiles" := Files, ~"SPDXID":=Id}) ->
+            lists:foreach(fun (#{~"hasFiles" := Files, ~"SPDXID":=Id}) ->
                               io:format("~p: length: ~p~n", [Id, length(Files)])
                       end, Packages),
             error(?FUNCTION_NAME)
@@ -1705,7 +1615,13 @@ test_download_location(#{~"packages" := Packages}) ->
 test_package_hasFiles(#{~"packages" := Packages}) ->
     %% test files are not repeated
     AllFiles = lists:foldl(fun (#{~"hasFiles" := FileIds}, Acc) -> FileIds ++ Acc end, [], Packages),
-    true = length(AllFiles) == length(lists:uniq(AllFiles)),
+
+    try
+        true = length(AllFiles) == length(lists:uniq(AllFiles))
+    catch _:_:_ ->
+            io:format("~p~n",[AllFiles -- lists:uniq(AllFiles)]),
+            error(?FUNCTION_NAME)
+    end,
 
     %% Test all files contain at least one file
     true = lists:all(fun (#{~"hasFiles" := Files}) -> erlang:length(Files) > 0 end, Packages),
@@ -1764,25 +1680,6 @@ test_has_extracted_licenses(#{~"hasExtractedLicensingInfos" := LicensesInfo,
     true = lists:all(fun (#{~"licenseId" := LicenseId}) -> lists:member(LicenseId, LicenseRefsInProject) end, LicensesInfo),
     ok.
 
-test_snippets(#{~"snippets" := Snippets, ~"files" := Files}=_Spdx) ->
-    true = lists:all(fun (#{ ~"SPDXID" := _Id,
-                         ~"copyrightText" := _Copyright,
-                         ~"licenseConcluded" := License,
-                         ~"licenseInfoInSnippets" := Licenses,
-                         ~"name" := _Name,
-                         ~"ranges" := [ #{~"endPointer" := #{ ~"lineNumber" := EndLine, ~"reference" := SpdxId},
-                                          ~"startPointer" := #{ ~"lineNumber" := StartLine, ~"reference" := SpdxId}},
-                                        #{~"endPointer" := #{ ~"offset" := EndOffsetBytes, ~"reference" := SpdxId},
-                                          ~"startPointer" := #{ ~"offset" := StartOffsetBytes, ~"reference" := SpdxId}} ],
-                         ~"snippetFromFile" := SpdxId}) ->
-                       EndLine >= StartLine andalso
-                      EndOffsetBytes >= StartOffsetBytes andalso
-                      lists:all(fun (L) -> lists:member(L, Licenses) end, split_licenses_in_individual_parts([License])) andalso
-                      length(lists:filter(fun (#{~"SPDXID" := Id}) -> SpdxId == Id end, Files)) == 1
-                     end, Snippets),
-    ok.
-
-
 %% Adds LicenseRef licenses where the text is missing.
 extracted_license_info() ->
     [begin

.github/workflows/main.yaml

@@ -62,6 +62,7 @@ jobs:
   pack:
     name: Build Erlang/OTP (64-bit)
     runs-on: ubuntu-latest
+    if: github.repository == 'erlang/otp' || github.event_name != 'scheduled'
     outputs:
         changes: ${{ steps.changes.outputs.changes }}
         build-c-code: ${{ steps.c-code-changes.outputs.changes != '[]' || env.FULL_BUILD_AND_CHECK == 'true' }}