Poetry: Support v1.5.0 or greater (#1420)

Author: meghfossa

Changelog.md

@@ -1,5 +1,8 @@
 # FOSSA CLI Changelog
 
+## v3.9.17
+- Poetry: Adds partial support for dependency groups. ([#1420](https://github.com/fossas/fossa-cli/pull/1420)). 
+
 ## v3.9.16
 - Treat `targets` field in the issue summary loaded from Core as optional during `fossa test` and `fossa report` ([#1422](https://github.com/fossas/fossa-cli/pull/1422)).
 - Adds support for SwiftPM v3 files ([#1424](https://github.com/fossas/fossa-cli/pull/1424)).
@@ -13,7 +16,7 @@
   SAAS customers are unaffected. ([#1418](https://github.com/fossas/fossa-cli/pull/1418)).
 
 ## v3.9.14
-- Update cargo strategy to parse new `cargo metadata` format for cargo >= 1.77.0 ([#1416](https://github.com/fossas/fossa-cli/pull/1416)).
+- Cargo: Update cargo strategy to parse new `cargo metadata` format for cargo >= 1.77.0 ([#1416](https://github.com/fossas/fossa-cli/pull/1416)).
 - `fossa release-group`: Add command to create a FOSSA release group release (`fossa release-group create-release`) [#1409](https://github.com/fossas/fossa-cli/pull/1409).
 - `fossa project`: Adds commands to interact with FOSSA projects (`fossa project edit`) [#1394](https://github.com/fossas/fossa-cli/pull/1395).
 

docs/references/strategies/languages/python/poetry.md

@@ -36,6 +36,7 @@ If `poetry.lock` file is not discovered, we fallback to reporting only direct de
 - For poetry project, build system's `build-backend` must be set to `poetry.core.masonry.api` or `poetry.masonry.api` in `pyproject.toml`. If not done so, it will not discover the project. Refer to [Poetry and PEP-517](https://python-poetry.org/docs/pyproject/#poetry-and-pep-517) for more details.
 - All extras specified in `[tool.poetry.extras]` are currently not reported.
 - Any [path dependencies](https://python-poetry.org/docs/dependency-specification/#path-dependencies) will not be reported.
+- For Poetry version greater or equal to `v1.5.0`, optional dependencies provideded in [dependencies group](https://python-poetry.org/docs/managing-dependencies/#dependency-groups) will not be included in the analysis, even with [--include-unused-deps](../../../subcommands/analyze.md), if only `pyproject.toml` is discovered.
 
 ## Example
 
@@ -181,6 +182,9 @@ _Dependencies highlighted in yellow boxes are direct dependencies, rest are tran
 
 Without `poetry.lock` we are not able to identify any transitive dependencies. We are also unable to locally resolve dependency when version ranges are provided, like `loguru = "^0.5"`.
 
+As `category` is not provided with poetry version greater or equal to [v1.5.0](https://github.com/dependabot/dependabot-core/pull/7418), FOSSA CLI will, first identify "main" dependencies by
+using `tool.poetry.dependencies` from `pyproject.toml`. Afterwhich, it will [hydrate](../../../../contributing/graph-hydration.md) dependencies. Any dependencies not hydrated, will be inferred to be a development dependency.
+
 ### References
 
 - [Poetry Source Code](https://github.com/python-poetry/poetry)

src/Strategy/Python/Poetry.hs

@@ -2,9 +2,13 @@ module Strategy.Python.Poetry (
   discover,
 
   -- * for testing only
-  graphFromLockFile,
-  setGraphDirectsFromPyproject,
+  findProjects,
+  analyze,
+  graphFromPyProjectAndLockFile,
   PoetryProject (..),
+  PyProjectTomlFile (..),
+  PoetryLockFile (..),
+  ProjectDir (..),
 ) where
 
 import App.Fossa.Analyze.Types (AnalyzeProject (analyzeProjectStaticOnly), analyzeProject)
@@ -17,8 +21,10 @@ import Data.Aeson (ToJSON)
 import Data.Map (Map)
 import Data.Map.Strict qualified as Map
 import Data.Maybe (fromMaybe)
+import Data.Set qualified as Set
 import Data.Text (Text)
-import DepTypes (DepType (..), Dependency (..))
+import Data.Text qualified as Text
+import DepTypes (DepEnvironment (EnvDevelopment), DepType (..), Dependency (..), hydrateDepEnvs)
 import Diag.Common (
   MissingDeepDeps (MissingDeepDeps),
   MissingEdges (MissingEdges),
@@ -40,9 +46,9 @@ import Strategy.Python.Errors (
   MissingPoetryLockFile (..),
   commitPoetryLockToVCS,
  )
-import Strategy.Python.Poetry.Common (getPoetryBuildBackend, logIgnoredDeps, pyProjectDeps, toCanonicalName, toMap)
-import Strategy.Python.Poetry.PoetryLock (PackageName (..), PoetryLock (..), PoetryLockPackage (..), poetryLockCodec)
-import Strategy.Python.Poetry.PyProject (PyProject (..), pyProjectCodec)
+import Strategy.Python.Poetry.Common (getPoetryBuildBackend, logIgnoredDeps, makePackageToLockDependencyMap, pyProjectDeps, toCanonicalName)
+import Strategy.Python.Poetry.PoetryLock (PackageName (..), PoetryLock (..), PoetryLockPackage (..), PoetryMetadata (poetryMetadataLockVersion), poetryLockCodec)
+import Strategy.Python.Poetry.PyProject (PyProject (..), allPoetryProductionDeps, pyProjectCodec)
 import Types (DependencyResults (..), DiscoveredProject (..), DiscoveredProjectType (PoetryProjectType), GraphBreadth (..))
 
 newtype PyProjectTomlFile = PyProjectTomlFile {pyProjectTomlPath :: Path Abs File} deriving (Eq, Ord, Show, Generic)
@@ -154,7 +160,7 @@ analyze PoetryProject{pyProjectToml, poetryLock} = do
     Just lockPath -> do
       poetryLockProject <- readContentsToml poetryLockCodec (poetryLockPath lockPath)
       _ <- logIgnoredDeps pyproject (Just poetryLockProject)
-      graph <- context "Building dependency graph from pyproject.toml and poetry.lock" $ pure $ setGraphDirectsFromPyproject (graphFromLockFile poetryLockProject) pyproject
+      graph <- context "Building dependency graph from pyproject.toml and poetry.lock" . pure $ graphFromPyProjectAndLockFile pyproject poetryLockProject
       pure $
         DependencyResults
           { dependencyGraph = graph
@@ -170,31 +176,55 @@ analyze PoetryProject{pyProjectToml, poetryLock} = do
         . errHelp MissingPoetryLockFileHelp
         . errDoc commitPoetryLockToVCS
         $ fatalText "poetry.lock file was not discovered"
-      graph <- context "Building dependency graph from only pyproject.toml" $ pure $ Graphing.fromList $ pyProjectDeps pyproject
+      graph <- context "Building dependency graph from only pyproject.toml" . pure $ Graphing.fromList $ pyProjectDeps pyproject
       pure $
         DependencyResults
           { dependencyGraph = graph
           , dependencyGraphBreadth = Partial
           , dependencyManifestFiles = [pyProjectTomlPath pyProjectToml]
           }
 
--- | Use a `pyproject.toml` to set the direct dependencies of a graph created from `poetry.lock`.
-setGraphDirectsFromPyproject :: Graphing Dependency -> PyProject -> Graphing Dependency
-setGraphDirectsFromPyproject graph pyproject = Graphing.promoteToDirect isDirect graph
-  where
-    -- Dependencies in `poetry.lock` are direct if they're specified in `pyproject.toml`.
-    -- `pyproject.toml` may use non canonical naming, when naming dependencies.
-    isDirect :: Dependency -> Bool
-    isDirect dep = case pyprojectPoetry pyproject of
-      Nothing -> False
-      Just _ -> any (\n -> toCanonicalName (dependencyName n) == toCanonicalName (dependencyName dep)) $ pyProjectDeps pyproject
-
 -- | Using a Poetry lockfile, build the graph of packages.
 -- The resulting graph contains edges, but does not distinguish between direct and deep dependencies,
 -- since `poetry.lock` does not indicate which dependencies are direct.
-graphFromLockFile :: PoetryLock -> Graphing Dependency
-graphFromLockFile poetryLock = Graphing.gmap pkgNameToDependency (edges <> Graphing.deeps pkgsNoDeps)
+graphFromPyProjectAndLockFile :: PyProject -> PoetryLock -> Graphing Dependency
+graphFromPyProjectAndLockFile pyProject poetryLock = graph
   where
+    -- Since Poetry lockfile v1.5 and gt, does not include category marker
+    -- it is not possible to know if the dependency is 'production' or 'development'.
+    -- strictly from lockfile itself. We hydrate "environment" from direct deps. If the
+    -- dependency has no environment, we mark it as dev environment. We know that all production
+    -- dependencies will be hydrated and will have some envionment.
+    graph :: Graphing Dependency
+    graph =
+      labelOptionalDepsIfPoetryGt1_5 $
+        hydrateDepEnvs $
+          Graphing.promoteToDirect isDirect $
+            Graphing.gmap pkgNameToDependency (edges <> Graphing.deeps pkgsNoDeps)
+
+    labelOptionalDepsIfPoetryGt1_5 :: Graphing Dependency -> Graphing Dependency
+    labelOptionalDepsIfPoetryGt1_5 g = if isLockLt1_5 then g else Graphing.gmap markEmptyEnvAsOptionalDep g
+
+    isLockLt1_5 :: Bool
+    isLockLt1_5 = any (`Text.isPrefixOf` lockVersion) ["0", "1.0", "1.1", "1.2", "1.3", "1.4"]
+
+    lockVersion :: Text
+    lockVersion = poetryMetadataLockVersion . poetryLockMetadata $ poetryLock
+
+    markEmptyEnvAsOptionalDep :: Dependency -> Dependency
+    markEmptyEnvAsOptionalDep d =
+      if null $ dependencyEnvironments d
+        then d{dependencyEnvironments = Set.singleton EnvDevelopment}
+        else d
+
+    directDeps :: [Dependency]
+    directDeps = pyProjectDeps pyProject
+
+    isDirect :: Dependency -> Bool
+    isDirect dep = case pyprojectPoetry pyProject of
+      Nothing -> False
+      Just _ -> any (\n -> toCanonicalName (dependencyName n) == toCanonicalName (dependencyName dep)) directDeps
+
     pkgs :: [PoetryLockPackage]
     pkgs = poetryLockPackages poetryLock
 
@@ -217,7 +247,10 @@ graphFromLockFile poetryLock = Graphing.gmap pkgNameToDependency (edges <> Graph
     canonicalPkgName name = PackageName . toCanonicalName $ unPackageName name
 
     mapOfDependency :: Map PackageName Dependency
-    mapOfDependency = toMap pkgs
+    mapOfDependency = makePackageToLockDependencyMap prodPkgNames pkgs
+
+    prodPkgNames :: [PackageName]
+    prodPkgNames = PackageName <$> Map.keys (allPoetryProductionDeps pyProject)
 
     -- Pip packages are [case insensitive](https://www.python.org/dev/peps/pep-0508/#id21), but poetry.lock may use
     -- non-canonical name for reference. Try to lookup with provided name, otherwise fallback to canonical naming.

src/Strategy/Python/Poetry/Common.hs

@@ -1,6 +1,6 @@
 module Strategy.Python.Poetry.Common (
   getPoetryBuildBackend,
-  toMap,
+  makePackageToLockDependencyMap,
   pyProjectDeps,
   logIgnoredDeps,
   toCanonicalName,
@@ -35,6 +35,7 @@ import Strategy.Python.Poetry.PyProject (
   PyProjectPoetryGitDependency (..),
   PyProjectPoetryPathDependency (..),
   PyProjectPoetryUrlDependency (..),
+  allPoetryNonProductionDeps,
   toDependencyVersion,
  )
 
@@ -63,7 +64,7 @@ logIgnoredDeps pyproject poetryLock = for_ notSupportedDepsMsgs (logDebug . pret
     notSupportedPyProjectDevDeps =
       Map.keys $
         Map.filter (not . supportedPyProjectDep) $
-          maybe Map.empty devDependencies (pyprojectPoetry pyproject)
+          allPoetryNonProductionDeps pyproject
 
     notSupportedPyProjectDeps :: [Text]
     notSupportedPyProjectDeps =
@@ -83,7 +84,28 @@ pyProjectDeps project = filter notNamedPython $ map snd allDeps
     notNamedPython = (/= "python") . dependencyName
 
     supportedDevDeps :: Map Text PoetryDependency
-    supportedDevDeps = Map.filter supportedPyProjectDep $ maybe Map.empty devDependencies (pyprojectPoetry project)
+    supportedDevDeps = Map.filter supportedPyProjectDep $ Map.unions [olderPoetryDevDeps, groupDeps]
+
+    -- These are dependencies coming from dev-dependencies table
+    -- which is pre 1.2.x style, understood by Poetry 1.0–1.2
+    olderPoetryDevDeps :: Map Text PoetryDependency
+    olderPoetryDevDeps = case pyprojectPoetry project of
+      Just (PyProjectPoetry{devDependencies}) -> devDependencies
+      _ -> mempty
+
+    -- These are 'group' dependencies. All group dependencies are optional.
+    -- Due to current toml parsing library limitation (specifically implicit table parsing support)
+    -- We only support dev, and test group. We may miss other development dependencies in our findings
+    -- if they are not named under 'dev' or 'test' group. This is not ideal, but is good partial solution
+    -- as optional deps are not included in the final analysis by default.
+    --
+    -- Refs:
+    -- \* https://github.com/kowainik/tomland/issues/336
+    -- \* https://python-poetry.org/docs/managing-dependencies#dependency-groups
+    groupDeps :: Map Text PoetryDependency
+    groupDeps = case pyprojectPoetry project of
+      Just (PyProjectPoetry{groupDevDependencies, groupTestDependencies}) -> Map.unions [groupDevDependencies, groupTestDependencies]
+      _ -> mempty
 
     supportedProdDeps :: Map Text PoetryDependency
     supportedProdDeps = Map.filter supportedPyProjectDep $ maybe Map.empty dependencies (pyprojectPoetry project)
@@ -154,11 +176,20 @@ toCanonicalName :: Text -> Text
 toCanonicalName t = toLower $ replace "_" "-" (replace "." "-" t)
 
 -- | Maps poetry lock package to map of package name and associated dependency.
-toMap :: [PoetryLockPackage] -> Map.Map PackageName Dependency
-toMap pkgs = Map.fromList $ (\x -> (canonicalPkgName x, toDependency x)) <$> (filter supportedPoetryLockDep pkgs)
+makePackageToLockDependencyMap :: [PackageName] -> [PoetryLockPackage] -> Map.Map PackageName Dependency
+makePackageToLockDependencyMap prodPkgs pkgs = Map.fromList $ (\x -> (lockCanonicalPackageName x, toDependency x)) <$> (filter supportedPoetryLockDep pkgs)
   where
-    canonicalPkgName :: PoetryLockPackage -> PackageName
-    canonicalPkgName pkg = PackageName $ toCanonicalName $ unPackageName $ poetryLockPackageName pkg
+    canonicalPkgName :: PackageName -> PackageName
+    canonicalPkgName = PackageName . toCanonicalName . unPackageName
+
+    lockCanonicalPackageName :: PoetryLockPackage -> PackageName
+    lockCanonicalPackageName = canonicalPkgName . poetryLockPackageName
+
+    canonicalProdPkgNames :: Set.Set PackageName
+    canonicalProdPkgNames = Set.fromList $ map canonicalPkgName prodPkgs
+
+    isProductionDirectDep :: PoetryLockPackage -> Bool
+    isProductionDirectDep pkg = lockCanonicalPackageName pkg `Set.member` canonicalProdPkgNames
 
     toDependency :: PoetryLockPackage -> Dependency
     toDependency pkg =
@@ -167,7 +198,7 @@ toMap pkgs = Map.fromList $ (\x -> (canonicalPkgName x, toDependency x)) <$> (fi
         , dependencyName = toDepName pkg
         , dependencyVersion = toDepVersion pkg
         , dependencyLocations = toDepLocs pkg
-        , dependencyEnvironments = Set.singleton $ toDepEnvironment pkg
+        , dependencyEnvironments = pkgEnvironments pkg
         , dependencyTags = Map.empty
         }
 
@@ -200,16 +231,19 @@ toMap pkgs = Map.fromList $ (\x -> (canonicalPkgName x, toDependency x)) <$> (fi
           ref <- poetryLockPackageSourceReference lockPkgSrc
           if poetryLockPackageSourceType lockPkgSrc /= "legacy" then Just ref else Nothing
 
-    toDepEnvironment :: PoetryLockPackage -> DepEnvironment
-    toDepEnvironment pkg = case poetryLockPackageCategory pkg of
+    pkgEnvironments :: PoetryLockPackage -> Set.Set DepEnvironment
+    pkgEnvironments pkg = case poetryLockPackageCategory pkg of
+      -- If category is provided, use category to infer if dependency's environment
       Just category -> case category of
-        "dev" -> EnvDevelopment
-        "main" -> EnvProduction
-        "test" -> EnvTesting
-        other -> EnvOther other
-      Nothing -> defaultDepEnvironment
-
-    defaultDepEnvironment :: DepEnvironment
-    -- Poetry made this field optional. When not present, it defaults to `main`, which maps to `EnvProduction`.
-    -- https://github.com/python-poetry/poetry/pull/7637
-    defaultDepEnvironment = EnvProduction
+        "dev" -> Set.singleton EnvDevelopment
+        "main" -> Set.singleton EnvProduction
+        "test" -> Set.singleton EnvTesting
+        other -> Set.singleton $ EnvOther other
+      -- If category is not provided, lockfile is likely greater than __.
+      -- In this case, if the package name exists in the dependencies
+      -- list, mark as production dependency, otherwise, mark it as development dependency
+      -- -
+      -- Refer to:
+      -- \* https://github.com/python-poetry/poetry/pull/7637
+      Nothing ->
+        (if isProductionDirectDep pkg then Set.singleton EnvProduction else mempty)

src/Strategy/Python/Poetry/PyProject.hs

@@ -9,16 +9,18 @@ module Strategy.Python.Poetry.PyProject (
   PyProjectPoetryGitDependency (..),
   PyProjectPoetryUrlDependency (..),
   PyProjectPoetryDetailedVersionDependency (..),
+  allPoetryProductionDeps,
 
   -- * for testing only
   parseConstraintExpr,
   toDependencyVersion,
+  allPoetryNonProductionDeps,
 ) where
 
 import Control.Monad.Combinators.Expr (Operator (..), makeExprParser)
 import Data.Foldable (asum)
 import Data.Functor (void)
-import Data.Map (Map)
+import Data.Map (Map, unions)
 import Data.Maybe (fromMaybe)
 import Data.String.Conversion (toString, toText)
 import Data.Text (Text)
@@ -113,10 +115,38 @@ data PyProjectPoetry = PyProjectPoetry
   , version :: Maybe Text
   , description :: Maybe Text
   , dependencies :: Map Text PoetryDependency
-  , devDependencies :: Map Text PoetryDependency
+  , -- For Poetry pre-1.2.x style, understood by Poetry 1.0–1.2
+    devDependencies :: Map Text PoetryDependency
+  , -- Since v1.2.0 of poetry, dependency groups are recommanded way to
+    -- provide development, test, and other optional dependencies.
+    -- refer to: https://python-poetry.org/docs/managing-dependencies#dependency-groups
+    --
+    -- Due to current toml-parsing limitations, we explicitly specify dev, and
+    -- test group only. Note that any dependencies from these groups are excluded
+    -- by default. refer to: https://github.com/kowainik/tomland/issues/336
+    groupDevDependencies :: Map Text PoetryDependency
+  , groupTestDependencies :: Map Text PoetryDependency
   }
   deriving (Show, Eq, Ord)
 
+allPoetryProductionDeps :: PyProject -> Map Text PoetryDependency
+allPoetryProductionDeps project = case pyprojectPoetry project of
+  Just (PyProjectPoetry{dependencies}) -> dependencies
+  _ -> mempty
+
+allPoetryNonProductionDeps :: PyProject -> Map Text PoetryDependency
+allPoetryNonProductionDeps project = unions [olderPoetryDevDeps, optionalDeps]
+  where
+    optionalDeps :: Map Text PoetryDependency
+    optionalDeps = case pyprojectPoetry project of
+      Just (PyProjectPoetry{groupDevDependencies, groupTestDependencies}) -> unions [groupDevDependencies, groupTestDependencies]
+      _ -> mempty
+
+    olderPoetryDevDeps :: Map Text PoetryDependency
+    olderPoetryDevDeps = case pyprojectPoetry project of
+      Just (PyProjectPoetry{devDependencies}) -> devDependencies
+      _ -> mempty
+
 data PoetryDependency
   = PoetryTextVersion Text
   | PyProjectPoetryDetailedVersionDependencySpec PyProjectPoetryDetailedVersionDependency
@@ -133,6 +163,8 @@ pyProjectPoetryCodec =
     <*> Toml.dioptional (Toml.text "description") .= description
     <*> Toml.tableMap Toml._KeyText pyProjectPoetryDependencyCodec "dependencies" .= dependencies
     <*> Toml.tableMap Toml._KeyText pyProjectPoetryDependencyCodec "dev-dependencies" .= devDependencies
+    <*> Toml.tableMap Toml._KeyText pyProjectPoetryDependencyCodec "group.dev.dependencies" .= groupDevDependencies
+    <*> Toml.tableMap Toml._KeyText pyProjectPoetryDependencyCodec "group.test.dependencies" .= groupTestDependencies
 
 pyProjectPoetryDependencyCodec :: Toml.Key -> TomlCodec PoetryDependency
 pyProjectPoetryDependencyCodec key =