Support true forced rebuilds for vendored dependencies, refactor types (#1423)

Author: jssblck

Changelog.md

@@ -1,7 +1,11 @@
 # FOSSA CLI Changelog
 
+## v3.9.18
+- Resolves an issue where `vendored-dependencies` were rescanned locally, but not in the FOSSA service,
+  when `forceRescans` was set to `true` ([#1423](https://github.com/fossas/fossa-cli/pull/1423)).
+
 ## v3.9.17
-- Poetry: Adds partial support for dependency groups. ([#1420](https://github.com/fossas/fossa-cli/pull/1420)). 
+- Poetry: Adds partial support for dependency groups. ([#1420](https://github.com/fossas/fossa-cli/pull/1420)).
 
 ## v3.9.16
 - Treat `targets` field in the issue summary loaded from Core as optional during `fossa test` and `fossa report` ([#1422](https://github.com/fossas/fossa-cli/pull/1422)).

integration-test/Analysis/LicenseScannerSpec.hs

@@ -9,7 +9,7 @@ import Analysis.FixtureUtils (
  )
 import App.Fossa.LicenseScanner (scanVendoredDep)
 import App.Fossa.VendoredDependency (VendoredDependency (VendoredDependency))
-import App.Types (FullFileUploads (..))
+import App.Types (FileUpload (..))
 import Control.Carrier.Diagnostics (runDiagnostics)
 import Control.Carrier.Stack (runStack)
 import Control.Carrier.StickyLogger (ignoreStickyLogger)
@@ -93,7 +93,7 @@ spec = do
     it "should find licenses in nested archives" $ do
       extractedDir <- getArtifact recursiveArchive
       let scanDir = extractedDir </> [reldir|cli-license-scan-integration-test-fixtures-main/recursive-archive|]
-      units <- runStack . runDiagnostics . ignoreStickyLogger . runExecIO . runReadFSIO . fmap licenseSourceUnitLicenseUnits $ scanVendoredDep scanDir Nothing (FullFileUploads False) vendoredDep
+      units <- runStack . runDiagnostics . ignoreStickyLogger . runExecIO . runReadFSIO . fmap licenseSourceUnitLicenseUnits $ scanVendoredDep scanDir Nothing FileUploadMatchData vendoredDep
       PIO.removeDirRecur extractedDir
       case units of
         Failure ws eg -> fail (show (renderFailure ws eg "An issue occurred"))
@@ -113,10 +113,10 @@ spec = do
             apacheUnit :: LicenseUnit
             apacheUnit = fromMaybe emptyLicenseUnit (head' $ NE.filter (\u -> licenseUnitName u == "apache-2.0") us)
 
-    it "should get full file contents from Themis if fullFileUploads is true" $ do
+    it "should get full file contents from Themis if full file uploads enabled" $ do
       extractedDir <- getArtifact recursiveArchive
       let scanDir = extractedDir </> [reldir|cli-license-scan-integration-test-fixtures-main/recursive-archive|]
-      units <- runStack . runDiagnostics . ignoreStickyLogger . runExecIO . runReadFSIO . fmap licenseSourceUnitLicenseUnits $ scanVendoredDep scanDir Nothing (FullFileUploads True) vendoredDep
+      units <- runStack . runDiagnostics . ignoreStickyLogger . runExecIO . runReadFSIO . fmap licenseSourceUnitLicenseUnits $ scanVendoredDep scanDir Nothing FileUploadFullContent vendoredDep
       PIO.removeDirRecur extractedDir
       case units of
         Failure ws eg -> fail (show (renderFailure ws eg "An issue occurred"))
@@ -140,7 +140,7 @@ spec = do
       extractedDir <- getArtifact recursiveArchive
       let scanDir = extractedDir </> [reldir|cli-license-scan-integration-test-fixtures-main/recursive-archive|]
       let licenseScanPathFilters = LicenseScanPathFilters{licenseScanPathFiltersOnly = [GlobFilter "**.rb"], licenseScanPathFiltersExclude = [], licenseScanPathFilterFileExclude = []}
-      units <- runStack . runDiagnostics . ignoreStickyLogger . runExecIO . runReadFSIO . fmap licenseSourceUnitLicenseUnits $ scanVendoredDep scanDir (Just licenseScanPathFilters) (FullFileUploads False) vendoredDep
+      units <- runStack . runDiagnostics . ignoreStickyLogger . runExecIO . runReadFSIO . fmap licenseSourceUnitLicenseUnits $ scanVendoredDep scanDir (Just licenseScanPathFilters) FileUploadMatchData vendoredDep
       PIO.removeDirRecur extractedDir
       case units of
         Failure ws eg -> fail (show (renderFailure ws eg "An issue occurred"))

src/App/Fossa/Analyze/Upload.hs

@@ -14,7 +14,7 @@ import App.Fossa.Reachability.Types (SourceUnitReachability)
 import App.Fossa.Reachability.Upload (upload)
 import App.Types (
   BaseDir (BaseDir),
-  FullFileUploads (FullFileUploads),
+  FileUpload,
   ProjectMetadata,
   ProjectRevision (..),
  )
@@ -61,7 +61,7 @@ import Effect.Logger (
   logStdout,
   viaShow,
  )
-import Fossa.API.Types (Organization (orgRequiresFullFileUploads, orgSupportsReachability, organizationId), Project (projectIsMonorepo), UploadResponse (..))
+import Fossa.API.Types (Organization (orgSupportsReachability, organizationId), Project (projectIsMonorepo), UploadResponse (..), orgFileUpload)
 import Path (Abs, Dir, Path)
 import Srclib.Types (
   FullSourceUnit,
@@ -126,13 +126,11 @@ uploadSuccessfulAnalysis (BaseDir basedir) metadata jsonOutput revision scanUnit
     uploadResult <- case scanUnits of
       SourceUnitOnly units -> uploadAnalysis revision metadata units
       LicenseSourceUnitOnly licenseSourceUnit -> do
-        let fullFileUploads = FullFileUploads $ orgRequiresFullFileUploads org
         let mergedUnits = mergeSourceAndLicenseUnits [] licenseSourceUnit
-        runStickyLogger SevInfo $ uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits fullFileUploads
+        runStickyLogger SevInfo . uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits $ orgFileUpload org
       SourceAndLicenseUnits sourceUnits licenseSourceUnit -> do
-        let fullFileUploads = FullFileUploads $ orgRequiresFullFileUploads org
         let mergedUnits = mergeSourceAndLicenseUnits sourceUnits licenseSourceUnit
-        runStickyLogger SevInfo $ uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits fullFileUploads
+        runStickyLogger SevInfo . uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits $ orgFileUpload org
 
     emitBuildWarnings uploadResult
 
@@ -167,11 +165,11 @@ uploadAnalysisWithFirstPartyLicensesToS3AndCore ::
   ProjectRevision ->
   ProjectMetadata ->
   NE.NonEmpty FullSourceUnit ->
-  FullFileUploads ->
+  FileUpload ->
   m UploadResponse
-uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits fullFileUploads = do
+uploadAnalysisWithFirstPartyLicensesToS3AndCore revision metadata mergedUnits uploadKind = do
   _ <- uploadAnalysisWithFirstPartyLicensesToS3 revision mergedUnits
-  uploadAnalysisWithFirstPartyLicenses revision metadata fullFileUploads
+  uploadAnalysisWithFirstPartyLicenses revision metadata uploadKind
 
 uploadAnalysisWithFirstPartyLicensesToS3 ::
   ( Has Diagnostics sig m

src/App/Fossa/ArchiveUploader.hs

@@ -11,6 +11,7 @@ import App.Fossa.VendoredDependency (
   dedupVendoredDeps,
   hashFile,
  )
+import App.Types (DependencyRebuild)
 import Control.Carrier.Diagnostics qualified as Diag
 import Control.Carrier.StickyLogger (StickyLogger, logSticky)
 import Control.Effect.Diagnostics (context)
@@ -69,17 +70,20 @@ compressAndUpload arcDir tmpDir VendoredDependency{..} = context "compressing an
 
 -- archiveUploadSourceUnit receives a list of vendored dependencies, a root path, and API settings.
 -- Using this information, it uploads each vendored dependency and queues a build for the dependency.
+--
+-- Note: this function intentionally does not accept a @FileUpload@ type, because it /always/ uploads full files.
 archiveUploadSourceUnit ::
   ( Has Diag.Diagnostics sig m
   , Has (Lift IO) sig m
   , Has StickyLogger sig m
   , Has Logger sig m
   , Has FossaApiClient sig m
   ) =>
+  DependencyRebuild ->
   Path Abs Dir ->
   NonEmpty VendoredDependency ->
   m (NonEmpty Locator)
-archiveUploadSourceUnit baseDir vendoredDeps = do
+archiveUploadSourceUnit rebuild baseDir vendoredDeps = do
   uniqDeps <- dedupVendoredDeps vendoredDeps
 
   -- At this point, we have a good list of deps, so go for it.
@@ -89,8 +93,7 @@ archiveUploadSourceUnit baseDir vendoredDeps = do
   -- orgID is appended when creating the build on the backend.  We don't care
   -- about the response here because if the build has already been queued, we
   -- get a 401 response.
-  res <- traverse queueArchiveBuild (NonEmpty.toList archives)
-  logDebug $ pretty $ show res
+  _ <- queueArchiveBuild (NonEmpty.toList archives) rebuild
 
   -- The organizationID is needed to prefix each locator name. The FOSSA API
   -- automatically prefixes the locator when queuing the build but not when

src/App/Fossa/FirstPartyScan.hs

@@ -8,7 +8,7 @@ module App.Fossa.FirstPartyScan (
 import App.Fossa.Config.Analyze (AnalyzeConfig (..), VendoredDependencyOptions (licenseScanPathFilters))
 import App.Fossa.LicenseScanner (scanVendoredDep)
 import App.Fossa.ManualDeps (ManualDependencies (vendoredDependencies), VendoredDependency (..), findAndReadFossaDepsFile)
-import App.Types (FirstPartyScansFlag (..), FullFileUploads (FullFileUploads))
+import App.Types (FirstPartyScansFlag (..))
 import Control.Carrier.Diagnostics qualified as Diag
 import Control.Carrier.FossaApiClient (runFossaApiClient)
 import Control.Effect.Debug (Debug)
@@ -25,7 +25,7 @@ import Diag.Result
 import Effect.Exec (Exec)
 import Effect.Logger (Logger, logDebug)
 import Effect.ReadFS (Has, ReadFS, resolvePath')
-import Fossa.API.Types (ApiOpts (..), Organization (..), blankOrganization)
+import Fossa.API.Types (ApiOpts (..), Organization (..), blankOrganization, orgFileUpload)
 import Path (Abs, Dir, Path, Rel, SomeBase (..))
 import Path.Extra
 import Path.IO
@@ -100,12 +100,12 @@ firstPartyScanMain base cfg org = do
   runFirstPartyScans <- shouldRunFirstPartyScans cfg org
   manualDeps <- findAndReadFossaDepsFile base
   let vdep = VendoredDependency "first-party" "." Nothing
-      fullFileUploads = FullFileUploads $ orgRequiresFullFileUploads org
+      uploadKind = orgFileUpload org
   pathFilters <- mergePathFilters base manualDeps (licenseScanPathFilters $ vendoredDeps cfg)
   case runFirstPartyScans of
     (True) -> do
       _ <- logDebug "Running a first-party license scan on the code in this repository. Licenses found in this repository will show up as 'Directly in code' in the FOSSA UI"
-      Just <$> scanVendoredDep base pathFilters fullFileUploads vdep
+      Just <$> scanVendoredDep base pathFilters uploadKind vdep
     (False) -> pure Nothing
 
 -- mergePathFilters takes the existing filters from the config and merges them with filters constructed by looking at the vendored dependencies

src/App/Fossa/Lernie/Analyze.hs

@@ -27,7 +27,7 @@ import App.Fossa.Lernie.Types (
   LernieWarning (..),
   OrgWideCustomLicenseConfigPolicy (..),
  )
-import App.Types (FullFileUploads (..))
+import App.Types (FileUpload (..))
 import Control.Carrier.Debug (Debug)
 import Control.Carrier.Diagnostics (Diagnostics, fatal, warn)
 import Control.Carrier.FossaApiClient (runFossaApiClient)
@@ -51,7 +51,7 @@ import Data.String.Conversion (ToText (toText), decodeUtf8)
 import Data.Text (Text)
 import Effect.Exec (AllowErr (Never), Command (..), Exec, execCurrentDirStdinThrow)
 import Effect.ReadFS (ReadFS)
-import Fossa.API.Types (ApiOpts, Organization (..))
+import Fossa.API.Types (ApiOpts, Organization (..), orgFileUpload)
 import Path (Abs, Dir, File, Path)
 import Srclib.Types (LicenseScanType (..), LicenseSourceUnit (..), LicenseUnit (..), LicenseUnitData (..), LicenseUnitInfo (..), LicenseUnitMatchData (..))
 
@@ -75,8 +75,8 @@ analyzeWithLernie ::
   m (Maybe LernieResults)
 analyzeWithLernie rootDir maybeApiOpts grepOptions = do
   case (maybeApiOpts, orgWideCustomLicenseScanConfigPolicy grepOptions) of
-    (_, Ignore) -> analyzeWithLernieMain rootDir grepOptions (FullFileUploads False)
-    (Nothing, Use) -> analyzeWithLernieMain rootDir grepOptions (FullFileUploads False)
+    (_, Ignore) -> analyzeWithLernieMain rootDir grepOptions FileUploadMatchData
+    (Nothing, Use) -> analyzeWithLernieMain rootDir grepOptions FileUploadMatchData
     (Just apiOpts, Use) -> runFossaApiClient apiOpts $ analyzeWithLernieWithOrgInfo rootDir grepOptions
 
 analyzeWithLernieWithOrgInfo ::
@@ -92,8 +92,10 @@ analyzeWithLernieWithOrgInfo ::
   m (Maybe LernieResults)
 analyzeWithLernieWithOrgInfo rootDir grepOptions = do
   orgWideCustomLicenses <- orgCustomLicenseScanConfigs <$> getOrganization
-  fullFiles <- orgRequiresFullFileUploads <$> getOrganization
-  analyzeWithLernieMain rootDir grepOptions{customLicenseSearch = nub $ orgWideCustomLicenses <> customLicenseSearch grepOptions} $ FullFileUploads fullFiles
+  uploadKind <- orgFileUpload <$> getOrganization
+
+  let options = grepOptions{customLicenseSearch = nub $ orgWideCustomLicenses <> customLicenseSearch grepOptions}
+  analyzeWithLernieMain rootDir options uploadKind
 
 analyzeWithLernieMain ::
   ( Has Diagnostics sig m
@@ -104,10 +106,10 @@ analyzeWithLernieMain ::
   ) =>
   Path Abs Dir ->
   GrepOptions ->
-  FullFileUploads ->
+  FileUpload ->
   m (Maybe LernieResults)
-analyzeWithLernieMain rootDir grepOptions fullFiles = do
-  let maybeLernieConfig = grepOptionsToLernieConfig rootDir grepOptions fullFiles
+analyzeWithLernieMain rootDir grepOptions uploadKind = do
+  let maybeLernieConfig = grepOptionsToLernieConfig rootDir grepOptions uploadKind
   case maybeLernieConfig of
     Just lernieConfig -> do
       unless (null $ customLicenseSearch grepOptions) $ trackUsage CustomLicenseSearchUsage
@@ -117,11 +119,13 @@ analyzeWithLernieMain rootDir grepOptions fullFiles = do
       pure $ Just lernieResults
     Nothing -> pure Nothing
 
-grepOptionsToLernieConfig :: Path Abs Dir -> GrepOptions -> FullFileUploads -> Maybe LernieConfig
-grepOptionsToLernieConfig rootDir grepOptions fullFiles =
+grepOptionsToLernieConfig :: Path Abs Dir -> GrepOptions -> FileUpload -> Maybe LernieConfig
+grepOptionsToLernieConfig rootDir grepOptions uploadKind =
   case (customLicenseSearches <> keywordSearches) of
     [] -> Nothing
-    res -> Just $ LernieConfig rootDir res $ unFullFileUploads fullFiles
+    res -> Just . LernieConfig rootDir res $ case uploadKind of
+      FileUploadMatchData -> False
+      FileUploadFullContent -> True
   where
     customLicenseSearches = map (grepEntryToLernieRegex CustomLicense) (customLicenseSearch grepOptions)
     keywordSearches = map (grepEntryToLernieRegex KeywordSearch) (keywordSearch grepOptions)

src/App/Fossa/LicenseScan.hs

@@ -21,7 +21,7 @@ import App.Fossa.VendoredDependency (
   VendoredDependency,
   dedupVendoredDeps,
  )
-import App.Types (BaseDir (BaseDir), FullFileUploads (FullFileUploads))
+import App.Types (BaseDir (BaseDir), FileUpload (FileUploadMatchData))
 import Control.Carrier.StickyLogger (
   Has,
   StickyLogger,
@@ -104,8 +104,7 @@ outputVendoredDeps (BaseDir dir) = runStickyLogger SevInfo $ do
   resultMap <- UploadUnits <$> runLicenseScan dir licenseScanPathFilters vendoredDeps
   logStdout . decodeUtf8 $ Aeson.encode resultMap
 
--- runLicenseScan does not require an API key, so we can't get the FullFileUploads param from the organization,
--- so we just default FullFileUploads to False.
+-- runLicenseScan does not require an API key, so we can't query the organization; default to uploading match data only.
 runLicenseScan ::
   ( Has Diagnostics sig m
   , Has ReadFS sig m
@@ -117,4 +116,4 @@ runLicenseScan ::
   Maybe LicenseScanPathFilters ->
   NonEmpty VendoredDependency ->
   m (NonEmpty LicenseSourceUnit)
-runLicenseScan basedir licenseScanPathFilters vdeps = dedupVendoredDeps vdeps >>= traverse (scanVendoredDep basedir licenseScanPathFilters $ FullFileUploads False)
+runLicenseScan basedir licenseScanPathFilters vdeps = dedupVendoredDeps vdeps >>= traverse (scanVendoredDep basedir licenseScanPathFilters FileUploadMatchData)