Add NOTICE files to source units (#1466)

Author: spatten

Changelog.md

@@ -1,9 +1,13 @@
 # FOSSA CLI Changelog
 
+## 3.9.37
+
+- License Scanning: Update Themis to include NOTICE files, and parse the additional NOTICE file fields in Themis's output. ([#1466](https://github.com/fossas/fossa-cli/pull/1466))
+
 ## 3.9.36
 
-- fossa-deps: Fixed an issue where Rocky Linux deps were not supported in the fossa-deps file ([#1473](https://github.com/fossas/fossa-cli/pull/1473))  
-- `fossa report`: Remove subscription type check in preflight checks ([#1474](https://github.com/fossas/fossa-cli/pull/1474))  
+- fossa-deps: Fixed an issue where Rocky Linux deps were not supported in the fossa-deps file ([#1473](https://github.com/fossas/fossa-cli/pull/1473))
+- `fossa report`: Remove subscription type check in preflight checks ([#1474](https://github.com/fossas/fossa-cli/pull/1474))
 
 ## 3.9.35
 

src/App/Fossa/Lernie/Analyze.hs

@@ -304,5 +304,6 @@ createLicenseUnitSingles ((path, title), licenseUnitData) =
         , licenseUnitDir = ""
         , licenseUnitFiles = NE.singleton (unCustomLicensePath path)
         , licenseUnitData = NE.singleton licenseUnitData
+        , licenseUnitNoticeFiles = []
         , licenseUnitInfo = LicenseUnitInfo{licenseUnitInfoDescription = Just $ "custom license search " <> unCustomLicenseTitle title}
         }

src/App/Fossa/ManualDeps.hs

@@ -206,6 +206,7 @@ toSourceUnit root depsFile manualDeps@ManualDependencies{..} maybeApiOpts vendor
       , sourceUnitType = "user-specific-yaml"
       , sourceUnitBuild = build
       , sourceUnitGraphBreadth = Complete
+      , sourceUnitNoticeFiles = []
       , sourceUnitOriginPaths = [someBaseToOriginPath originPath]
       , additionalData = additional
       }
@@ -508,7 +509,8 @@ instance FromJSON CustomDependency where
       <$> (obj `neText` "name")
       <*> (unTextLike <$> obj `neText` "version")
       <*> (obj `neText` "license")
-      <*> obj .:? "metadata"
+      <*> obj
+        .:? "metadata"
       <* forbidMembers "custom dependencies" ["type", "path", "url"] obj
 
 instance FromJSON RemoteDependency where

src/Srclib/Converter.hs

@@ -71,6 +71,7 @@ toSourceUnit leaveUnfiltered path dependencies projectType graphBreadth originPa
             , buildDependencies = deps
             }
     , sourceUnitGraphBreadth = graphBreadth
+    , sourceUnitNoticeFiles = []
     , sourceUnitOriginPaths = originPaths
     , additionalData = Nothing
     }

src/Srclib/Types.hs

@@ -5,6 +5,7 @@ module Srclib.Types (
   SourceUnit (..),
   SourceUnitBuild (..),
   SourceUnitDependency (..),
+  SourceUnitNoticeFile (..),
   AdditionalDepData (..),
   SourceUserDefDep (..),
   SourceRemoteDep (..),
@@ -112,13 +113,36 @@ textToOriginPath = OriginPath . toString
 --   Manifest?: string;
 -- }
 
+data SourceUnitNoticeFile = SourceUnitNoticeFile
+  { sourceUnitNoticeFilePath :: Text
+  , sourceUnitNoticeFileContents :: Text
+  , sourceUnitNoticeFileCopyrights :: [Text]
+  }
+  deriving (Eq, Ord, Show)
+
+instance ToJSON SourceUnitNoticeFile where
+  toJSON SourceUnitNoticeFile{..} =
+    object
+      [ "path" .= sourceUnitNoticeFilePath
+      , "contents" .= sourceUnitNoticeFileContents
+      , "copyrights" .= sourceUnitNoticeFileCopyrights
+      ]
+
+instance FromJSON SourceUnitNoticeFile where
+  parseJSON = withObject "SourceUnitNoticeFile" $ \obj ->
+    SourceUnitNoticeFile
+      <$> obj .: "path"
+      <*> obj .: "contents"
+      <*> obj .:? "copyrights" .!= []
+
 data FullSourceUnit = FullSourceUnit
   { fullSourceUnitName :: Text
   , fullSourceUnitType :: Text
   , fullSourceUnitTitle :: Maybe Text
   , fullSourceUnitManifest :: Maybe Text
   , fullSourceUnitBuild :: Maybe SourceUnitBuild
   , fullSourceUnitGraphBreadth :: GraphBreadth
+  , fullSourceUnitNoticeFiles :: [SourceUnitNoticeFile]
   , fullSourceUnitOriginPaths :: [OriginPath]
   , fullSourceUnitAdditionalData :: Maybe AdditionalDepData
   , fullSourceUnitFiles :: Maybe (NonEmpty Text)
@@ -136,6 +160,7 @@ licenseUnitToFullSourceUnit LicenseUnit{..} =
     , fullSourceUnitManifest = Nothing
     , fullSourceUnitBuild = Nothing
     , fullSourceUnitGraphBreadth = Complete
+    , fullSourceUnitNoticeFiles = licenseUnitNoticeFiles
     , fullSourceUnitOriginPaths = []
     , fullSourceUnitAdditionalData = Nothing
     , fullSourceUnitFiles = Just licenseUnitFiles
@@ -153,6 +178,7 @@ sourceUnitToFullSourceUnit SourceUnit{..} =
     , fullSourceUnitBuild = sourceUnitBuild
     , fullSourceUnitGraphBreadth = sourceUnitGraphBreadth
     , fullSourceUnitOriginPaths = sourceUnitOriginPaths
+    , fullSourceUnitNoticeFiles = sourceUnitNoticeFiles
     , fullSourceUnitAdditionalData = additionalData
     , fullSourceUnitFiles = Nothing
     , fullSourceUnitData = Nothing
@@ -169,6 +195,7 @@ instance ToJSON FullSourceUnit where
       , "Build" .= fullSourceUnitBuild
       , "GraphBreadth" .= fullSourceUnitGraphBreadth
       , "OriginPaths" .= fullSourceUnitOriginPaths
+      , "NoticeFiles" .= fullSourceUnitNoticeFiles
       , "AdditionalDependencyData" .= fullSourceUnitAdditionalData
       , "Files" .= fullSourceUnitFiles
       , "Data" .= fullSourceUnitData
@@ -204,6 +231,7 @@ data LicenseUnit = LicenseUnit
   , licenseUnitDir :: Text
   , licenseUnitFiles :: (NonEmpty Text)
   , licenseUnitData :: (NonEmpty LicenseUnitData)
+  , licenseUnitNoticeFiles :: [SourceUnitNoticeFile]
   , licenseUnitInfo :: LicenseUnitInfo
   }
   deriving (Eq, Ord, Show)
@@ -217,6 +245,7 @@ emptyLicenseUnit =
     , licenseUnitDir = ""
     , licenseUnitFiles = "" :| []
     , licenseUnitData = emptyLicenseUnitData :| []
+    , licenseUnitNoticeFiles = []
     , licenseUnitInfo = LicenseUnitInfo{licenseUnitInfoDescription = Nothing}
     }
 
@@ -235,6 +264,7 @@ instance ToJSON LicenseUnit where
       , "Dir" .= licenseUnitDir
       , "Files" .= licenseUnitFiles
       , "Data" .= licenseUnitData
+      , "NoticeFiles" .= licenseUnitNoticeFiles
       , "Info" .= licenseUnitInfo
       ]
 
@@ -247,6 +277,7 @@ instance FromJSON LicenseUnit where
       <*> obj .: "Dir"
       <*> obj .: "Files"
       <*> obj .: "Data"
+      <*> obj .:? "NoticeFiles" .!= []
       <*> obj .: "Info"
 
 newtype LicenseUnitInfo = LicenseUnitInfo
@@ -350,6 +381,7 @@ data SourceUnit = SourceUnit
   -- ^ path to manifest file
   , sourceUnitBuild :: Maybe SourceUnitBuild
   , sourceUnitGraphBreadth :: GraphBreadth
+  , sourceUnitNoticeFiles :: [SourceUnitNoticeFile]
   , sourceUnitOriginPaths :: [OriginPath]
   , additionalData :: Maybe AdditionalDepData
   }
@@ -431,6 +463,7 @@ instance ToJSON SourceUnit where
       , "Manifest" .= sourceUnitManifest
       , "Build" .= sourceUnitBuild
       , "GraphBreadth" .= sourceUnitGraphBreadth
+      , "NoticeFiles" .= sourceUnitNoticeFiles
       , "OriginPaths" .= sourceUnitOriginPaths
       , "AdditionalDependencyData" .= additionalData
       ]
@@ -443,6 +476,7 @@ instance FromJSON SourceUnit where
       <*> obj .: "Manifest"
       <*> obj .:? "Build"
       <*> obj .: "GraphBreadth"
+      <*> obj .:? "NoticeFiles" .!= []
       <*> obj .: "OriginPaths"
       <*> obj .:? "AdditionalDependencyData"
 

test/App/Fossa/Analyze/UploadSpec.hs

@@ -88,6 +88,7 @@ expectedMergedFullSourceUnits = NE.fromList [fullSourceUnit, fullLicenseUnit]
         , fullSourceUnitFiles = Nothing
         , fullSourceUnitData = Nothing
         , fullSourceUnitInfo = Nothing
+        , fullSourceUnitNoticeFiles = []
         }
     fullLicenseUnit =
       FullSourceUnit
@@ -102,6 +103,7 @@ expectedMergedFullSourceUnits = NE.fromList [fullSourceUnit, fullLicenseUnit]
         , fullSourceUnitFiles = Just $ "" NE.:| []
         , fullSourceUnitData = Just $ emptyLicenseUnitData NE.:| []
         , fullSourceUnitInfo = Just LicenseUnitInfo{licenseUnitInfoDescription = Nothing}
+        , fullSourceUnitNoticeFiles = []
         }
 
 expectGetSuccessWithReachability :: Has MockApi sig m => m ()

test/App/Fossa/Container/AnalyzeNativeSpec.hs

@@ -236,6 +236,7 @@ jarsInContainerSpec = describe "Jars in Containers" $ do
                     }
             , sourceUnitGraphBreadth = Complete
             , sourceUnitOriginPaths = [textToOriginPath "package-lock.json"]
+            , sourceUnitNoticeFiles = []
             , additionalData = Nothing
             }
 

test/App/Fossa/FirstPartyScanSpec.hs

@@ -10,10 +10,11 @@ import Control.Effect.FossaApiClient (FossaApiClientF (..))
 import Data.List qualified as List
 import Data.List.NonEmpty qualified as NE
 import Data.Maybe (isJust)
+import Data.Text (strip)
 import Fossa.API.Types (Organization (..))
 import Path (Dir, Path, Rel, mkRelDir, (</>))
 import Path.IO (getCurrentDir)
-import Srclib.Types (LicenseSourceUnit (..), LicenseUnit (licenseUnitData, licenseUnitName), LicenseUnitData (licenseUnitDataContents))
+import Srclib.Types (LicenseSourceUnit (..), LicenseUnit (..), LicenseUnitData (licenseUnitDataContents), SourceUnitNoticeFile (..))
 import Test.Effect (expectFatal', expectationFailure', it', shouldBe')
 import Test.Fixtures qualified as Fixtures
 import Test.Hspec (Spec, describe, runIO)
@@ -25,12 +26,16 @@ fixtureDir = $(mkRelDir "test/App/Fossa/VendoredDependency/testdata/repo")
 fixtureDirWithVendoredDeps :: Path Rel Dir
 fixtureDirWithVendoredDeps = $(mkRelDir "test/App/Fossa/VendoredDependency/testdata/firstparty")
 
+fixtureDirWithNoticeFiles :: Path Rel Dir
+fixtureDirWithNoticeFiles = $(mkRelDir "test/App/Fossa/VendoredDependency/testdata/firstparty-with-notice")
+
 spec :: Spec
 spec = do
   describe "runFirstPartyScan" $ do
     currDir <- runIO getCurrentDir
     let scanDir = currDir </> fixtureDir
     let scanDirWithVendoredDeps = currDir </> fixtureDirWithVendoredDeps
+    let scanDirWithNoticeFiles = currDir </> fixtureDirWithNoticeFiles
 
     it' "should fail if the organization does not support first party scans and you force it on" $ do
       expectGetOrganizationThatDoesNotSupportFirstPartyScans
@@ -99,6 +104,26 @@ spec = do
           licenseUnitName secondUnit `shouldBe'` "apache-2.0"
           licenseUnitDataContents unitData `shouldBe'` Nothing
 
+    it' "should find notice files" $ do
+      expectGetOrganizationThatDefaultsToFirstPartyScans
+      licenseSourceUnit <- firstPartyScanWithOrgInfo scanDirWithNoticeFiles Fixtures.standardAnalyzeConfig
+      case licenseSourceUnit of
+        Nothing -> expectationFailure' "first party scan should have run"
+        Just LicenseSourceUnit{licenseSourceUnitLicenseUnits = units} -> do
+          length units `shouldBe'` 3
+          let noticeUnit = NE.head units
+          licenseUnitName noticeUnit `shouldBe'` ""
+          licenseUnitType noticeUnit `shouldBe'` "NoticeFileMatches"
+          let noticeFiles = licenseUnitNoticeFiles noticeUnit
+          length noticeFiles `shouldBe'` 1
+          let noticeFile = head noticeFiles
+          let copyrights = sourceUnitNoticeFileCopyrights noticeFile
+          length copyrights `shouldBe'` 1
+          let copyright = head copyrights
+          copyright `shouldBe'` "2024 Frank Frankson"
+          strip (sourceUnitNoticeFileContents noticeFile) `shouldBe'` "This is a notice file that is copyright 2024 Frank Frankson"
+          sourceUnitNoticeFilePath noticeFile `shouldBe'` "NOTICE.txt"
+
 -- The default org defaults to not running first party scans but has first-party scans enabled
 expectGetOrganizationThatDefaultsToNoFirstPartyScans :: Has MockApi sig m => m ()
 expectGetOrganizationThatDefaultsToNoFirstPartyScans = GetOrganization `alwaysReturns` Fixtures.organization

test/App/Fossa/LernieSpec.hs

@@ -172,6 +172,7 @@ expectedLicenseUnit =
     , licenseUnitFiles = NE.singleton $ toText . toFilePath $ absDir </> $(mkRelDir "two.txt")
     , licenseUnitData = NE.singleton expectedUnitData
     , licenseUnitInfo = LicenseUnitInfo{licenseUnitInfoDescription = Just "custom license search Proprietary License"}
+    , licenseUnitNoticeFiles = []
     }
 
 expectedDoubleSourceUnit :: LicenseSourceUnit

test/App/Fossa/LicenseScannerSpec.hs

@@ -43,6 +43,7 @@ unitOne =
     , licenseUnitFiles =
         NE.fromList ["foo/bar/LICENSE", "foo/bar/one.txt"]
     , licenseUnitInfo = info
+    , licenseUnitNoticeFiles = []
     }
 
 unitTwo :: LicenseUnit
@@ -55,6 +56,7 @@ unitTwo =
     , licenseUnitData = NE.fromList [emptyLicenseUnitData{licenseUnitDataPath = "foo/bar/baz/ANOTHER_LICENSE"}, emptyLicenseUnitData{licenseUnitDataPath = "foo/bar/baz/two.txt"}]
     , licenseUnitFiles = NE.fromList ["foo/bar/baz/ANOTHER_LICENSE", "foo/bar/baz/two.txt"]
     , licenseUnitInfo = info
+    , licenseUnitNoticeFiles = []
     }
 expectedCombinedUnit :: LicenseUnit
 expectedCombinedUnit =
@@ -78,6 +80,7 @@ expectedCombinedUnit =
           , "foo/bar/one.txt"
           ]
     , licenseUnitInfo = info
+    , licenseUnitNoticeFiles = []
     }
 
 fixtureDir :: Path Rel Dir