[ANE-1070] Pass licenseScanPathFilters to lernie

Author: james-fossa

src/App/Fossa/Analyze.hs

@@ -338,7 +338,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do
         then do
           logInfo "Running in VSI only mode, skipping keyword search and custom-license search"
           pure Nothing
-        else Diag.context "custom-license & keyword search" . runStickyLogger SevInfo $ analyzeWithLernie basedir maybeApiOpts grepOptions
+        else Diag.context "custom-license & keyword search" . runStickyLogger SevInfo $ analyzeWithLernie basedir maybeApiOpts grepOptions $ Config.licenseScanPathFilters vendoredDepsOptions
   let lernieResults = join . resultToMaybe $ maybeLernieResults
 
   let -- This makes nice with additionalSourceUnits below, but throws out additional Result data.

src/App/Fossa/Config/Common.hs

@@ -286,7 +286,8 @@ collectBaseFile ::
   , Has (Lift IO) sig m
   , Has ReadFS sig m
   ) =>
-  FilePath -> m (Path Abs File)
+  FilePath ->
+  m (Path Abs File)
 collectBaseFile = validateFile
 
 validateDir ::

src/App/Fossa/Config/Report.hs

@@ -243,7 +243,9 @@ mergeOpts cfgfile envvars ReportCliOptions{..} = do
       , Has ReadFS sig m
       , Has Diagnostics sig m
       ) =>
-      Text -> OverrideProject -> m (ProjectRevision, ReportBase)
+      Text ->
+      OverrideProject ->
+      m (ProjectRevision, ReportBase)
     generateDirOrSBOMBase path projectOverride = do
       basedir <- Diag.recover $ collectBaseDir (Conv.toString path)
       case basedir of

src/App/Fossa/Lernie/Analyze.hs

@@ -54,6 +54,7 @@ import Effect.ReadFS (ReadFS)
 import Fossa.API.Types (ApiOpts, Organization (..), orgFileUpload)
 import Path (Abs, Dir, File, Path)
 import Srclib.Types (LicenseScanType (..), LicenseSourceUnit (..), LicenseUnit (..), LicenseUnitData (..), LicenseUnitInfo (..), LicenseUnitMatchData (..))
+import Types (LicenseScanPathFilters)
 
 newtype CustomLicensePath = CustomLicensePath {unCustomLicensePath :: Text}
   deriving (Eq, Ord, Show, Hashable)
@@ -72,12 +73,13 @@ analyzeWithLernie ::
   Path Abs Dir ->
   Maybe ApiOpts ->
   GrepOptions ->
+  Maybe LicenseScanPathFilters ->
   m (Maybe LernieResults)
-analyzeWithLernie rootDir maybeApiOpts grepOptions = do
+analyzeWithLernie rootDir maybeApiOpts grepOptions filters = do
   case (maybeApiOpts, orgWideCustomLicenseScanConfigPolicy grepOptions) of
-    (_, Ignore) -> analyzeWithLernieMain rootDir grepOptions FileUploadMatchData
-    (Nothing, Use) -> analyzeWithLernieMain rootDir grepOptions FileUploadMatchData
-    (Just apiOpts, Use) -> runFossaApiClient apiOpts $ analyzeWithLernieWithOrgInfo rootDir grepOptions
+    (_, Ignore) -> analyzeWithLernieMain rootDir grepOptions filters FileUploadMatchData
+    (Nothing, Use) -> analyzeWithLernieMain rootDir grepOptions filters FileUploadMatchData
+    (Just apiOpts, Use) -> runFossaApiClient apiOpts $ analyzeWithLernieWithOrgInfo rootDir grepOptions filters
 
 analyzeWithLernieWithOrgInfo ::
   ( Has Diagnostics sig m
@@ -89,14 +91,15 @@ analyzeWithLernieWithOrgInfo ::
   ) =>
   Path Abs Dir ->
   GrepOptions ->
+  Maybe LicenseScanPathFilters ->
   m (Maybe LernieResults)
-analyzeWithLernieWithOrgInfo rootDir grepOptions = do
+analyzeWithLernieWithOrgInfo rootDir grepOptions filters = do
   org <- getOrganization
   let orgWideCustomLicenses = orgCustomLicenseScanConfigs org
       uploadKind = orgFileUpload org
 
   let options = grepOptions{customLicenseSearch = nub $ orgWideCustomLicenses <> customLicenseSearch grepOptions}
-  analyzeWithLernieMain rootDir options uploadKind
+  analyzeWithLernieMain rootDir options filters uploadKind
 
 analyzeWithLernieMain ::
   ( Has Diagnostics sig m
@@ -107,10 +110,11 @@ analyzeWithLernieMain ::
   ) =>
   Path Abs Dir ->
   GrepOptions ->
+  Maybe LicenseScanPathFilters ->
   FileUpload ->
   m (Maybe LernieResults)
-analyzeWithLernieMain rootDir grepOptions uploadKind = do
-  let maybeLernieConfig = grepOptionsToLernieConfig rootDir grepOptions uploadKind
+analyzeWithLernieMain rootDir grepOptions filters uploadKind = do
+  let maybeLernieConfig = grepOptionsToLernieConfig rootDir grepOptions filters uploadKind
   case maybeLernieConfig of
     Just lernieConfig -> do
       unless (null $ customLicenseSearch grepOptions) $ trackUsage CustomLicenseSearchUsage
@@ -120,11 +124,11 @@ analyzeWithLernieMain rootDir grepOptions uploadKind = do
       pure $ Just lernieResults
     Nothing -> pure Nothing
 
-grepOptionsToLernieConfig :: Path Abs Dir -> GrepOptions -> FileUpload -> Maybe LernieConfig
-grepOptionsToLernieConfig rootDir grepOptions uploadKind =
+grepOptionsToLernieConfig :: Path Abs Dir -> GrepOptions -> Maybe LicenseScanPathFilters -> FileUpload -> Maybe LernieConfig
+grepOptionsToLernieConfig rootDir grepOptions filters uploadKind =
   case (customLicenseSearches <> keywordSearches) of
     [] -> Nothing
-    res -> Just . LernieConfig rootDir res $ case uploadKind of
+    res -> Just . LernieConfig rootDir res filters $ case uploadKind of
       FileUploadMatchData -> False
       FileUploadFullContent -> True
   where

src/App/Fossa/Lernie/Types.hs

@@ -24,6 +24,7 @@ import Data.Text (Text)
 import GHC.Generics (Generic)
 import Path (Abs, Dir, File, Path)
 import Srclib.Types (LicenseSourceUnit)
+import Types (LicenseScanPathFilters (..))
 
 data OrgWideCustomLicenseConfigPolicy = Use | Ignore
   deriving (Eq, Ord, Show)
@@ -72,6 +73,7 @@ instance FromJSON GrepEntry where
 data LernieConfig = LernieConfig
   { rootDir :: Path Abs Dir
   , regexes :: [LernieRegex]
+  , licenseScanPathFilters :: Maybe LicenseScanPathFilters
   , fullFiles :: Bool
   }
   deriving (Eq, Ord, Show, Generic)
@@ -81,6 +83,7 @@ instance ToJSON LernieConfig where
     object
       [ "root_dir" .= toText rootDir
       , "regexes" .= toJSON regexes
+      , "license_scan_path_filters" .= toJSON licenseScanPathFilters
       , "full_files" .= fullFiles
       ]
 

test/App/Fossa/LernieSpec.hs

@@ -26,6 +26,7 @@ import Test.Effect (expectationFailure', it', shouldBe')
 import Test.Fixtures qualified as Fixtures
 import Test.Hspec (Spec, describe, it, runIO, shouldBe)
 import Test.MockApi (alwaysReturns)
+import Types (GlobFilter (GlobFilter), LicenseScanPathFilters (..))
 
 customLicenseLernieMatchData :: LernieMatchData
 customLicenseLernieMatchData =
@@ -264,6 +265,13 @@ expectedLernieConfig =
   LernieConfig
     { rootDir = absDir
     , regexes = [customLicenseLernieRegex, keywordSearchLernieRegex]
+    , licenseScanPathFilters =
+        Just
+          LicenseScanPathFilters
+            { licenseScanPathFiltersOnly = [GlobFilter "/*", GlobFilter "/**"]
+            , licenseScanPathFiltersExclude = [GlobFilter "/*", GlobFilter "/**"]
+            , licenseScanPathFilterFileExclude = []
+            }
     , fullFiles = False
     }