add support for pip

Author: righettod

scripts/generate-valid-project-descriptor.py

@@ -6,6 +6,7 @@
 - OWASP Dependency Check: https://owasp.org/www-project-dependency-check/
 - NPM audit: https://docs.npmjs.com/cli/v9/commands/npm-audit
 - dotnet package list: https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-package-list
+- PIP audit: https://pypi.org/project/pip-audit/
 
 For Maven:
     This script analyse all the "pom.xml" descriptor files present recursively in a folder and
@@ -31,7 +32,14 @@
     Once generated, use the command "dotnet restore --verbosity quiet" into the folder where the "project.csproj" file was created
     to gather all dependencies.
 
-It leverage the data provided by the Google site "https://deps.dev/".
+For Pip:
+    The management of dependencies via the file "pyproject.toml" is used as reference as it is the modern way to manage dependencies in a project (See https://peps.python.org/pep-0621/).
+    This script analyse all the "pyproject.toml" descriptor files present recursively in a folder and create a single requirements file named "requirements-consolidated.txt" with all dependencies resolvable via the online official registry.
+    Extract dependencies from "[project] > dependencies" only.
+    The generated file can be provided to "pip-audit" as the source of dependencies to analyze.
+
+
+It leverage the data provided by the Google site "https://deps.dev/" to identify if a dependency exist into the "online official registry".
 
 Other type of project will be added, over the time, based on the case I meet :)
 
@@ -42,9 +50,14 @@
 import argparse
 import pathlib
 import json
+import re
+import tomllib
 import xml.etree.ElementTree as ET
 from termcolor import colored
 
+# KEY is the package manager official name and VALUE if the corresponding name used by deps.dev
+DEPSDEV_PACKAGE_MANAGER_MAPPING = {"pip": "pypi"}
+
 MAVEN_PROJECT_DESCRIPTOR_TPL = """<?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 	<modelVersion>4.0.0</modelVersion>
@@ -241,6 +254,21 @@ def list_packages_nuget(xml_content):
     return dependencies
 
 
+def list_packages_pip(content):
+    data = tomllib.loads(content)
+    project_data = data.get("project", {})
+    dependencies_data = project_data.get("dependencies", [])
+    dependencies = []
+    for dependency_data in dependencies_data:
+        name = re.findall(r'([a-z0-9_\.\-\[\]]+)[<>=~!^]', dependency_data, re.IGNORECASE)[0]
+        name = name.strip("<>=~!^")
+        version = dependency_data.replace(name, "")
+        pkg_name = f"{name}:{version}"
+        if pkg_name not in dependencies:
+            dependencies.append(pkg_name)
+    return dependencies
+
+
 def encode_package_name(pkg_name):
     name_encoded = pkg_name.replace("/", "%2F").replace("@", "%40").replace(":", "%3A")
     return name_encoded
@@ -266,13 +294,25 @@ def find_package_present_into_registry(pkg_names_list, project_type):
                 pkg_name_only = parts[0]
                 version_only = parts[1].split(" ")[0].strip("<>=^|")
                 name_encoded = encode_package_name(pkg_name_only)
+            if project_type == "pip":
+                parts = pkg_name.split(":")
+                pkg_name_only = parts[0]
+                version_only = parts[1].split(",")[0].strip("<>=~!^")
+                name_encoded = encode_package_name(pkg_name_only)
             if project_type == "nuget":
                 parts = pkg_name.split(":")
                 pkg_name_only = parts[0]
                 version_only = parts[1]
                 name_encoded = encode_package_name(pkg_name_only)
             if name_encoded is not None and version_only is not None:
-                u = f"https://deps.dev/_/s/{project_type}/p/{name_encoded}/v/{version_only}"
+                project_type_target = project_type
+                if project_type in DEPSDEV_PACKAGE_MANAGER_MAPPING:
+                    project_type_target = DEPSDEV_PACKAGE_MANAGER_MAPPING[project_type]
+                u = f"https://deps.dev/_/s/{project_type_target}/p/{name_encoded}/v/{version_only}"
+                # With python (pip) sometime a package is specified via "3.10" instead of "3.10.0".
+                # So I need perform a check without the version to just check the presence of the package.
+                if project_type == "pip":
+                    u = f"https://deps.dev/_/s/{project_type_target}/p/{name_encoded}"
                 response = session.get(url=u)
                 if response.status_code == 200 and "version" in response.json():
                     present.append(pkg_name)
@@ -295,7 +335,7 @@ def generate_project_descriptor(project_descriptor_file_name, pkg_names_list, gl
     if project_type == "npm":
         # Handle the prevention of adding several times the same package at this level as it is the final
         # step where all dependencies were identified.
-        # It is more important in NPM (as compared to MAVEN) as a range of versions is specified for a package.
+        # It is more important in NPM (as compared to MAVEN) as a range of versions can be specified for a package.
         packages_already_added = []
         for pkg_name in pkg_names_list:
             parts = pkg_name.split(":")
@@ -304,6 +344,19 @@ def generate_project_descriptor(project_descriptor_file_name, pkg_names_list, gl
                 dependencies.append(NPM_DEPENDENCY_TPL % (parts[0], parts[1]))
                 packages_already_added.append(pkg_name_only)
         project_descriptor_content = NPM_PROJECT_DESCRIPTOR_TPL % (",".join(dependencies))
+    if project_type == "pip":
+        # Handle the prevention of adding several times the same package at this level as it is the final
+        # step where all dependencies were identified.
+        # It is more important in PIP (as compared to MAVEN) as a range of versions can be specified for a package.
+        packages_already_added = []
+        for pkg_name in pkg_names_list:
+            parts = pkg_name.split(":")
+            pkg_name_only = parts[0].lower()
+            if pkg_name_only not in packages_already_added:
+                dependencies.append(f"{parts[0]}{parts[1]}")
+                packages_already_added.append(pkg_name_only)
+        project_descriptor_content = "\n".join(dependencies)
+        project_descriptor_target_file_name = "requirements-consolidated.txt"
     if project_type == "nuget":
         for pkg_name in pkg_names_list:
             parts = pkg_name.split(":")
@@ -324,7 +377,7 @@ def is_npm_package_json_file(file_content):
     parser = argparse.ArgumentParser(description="Generate a valid project descriptor with all dependencies resolvable via online official registry.")
     required_params = parser.add_argument_group("required named arguments")
     required_params.add_argument("-f", action="store", dest="base_folder", help="Path to folder containing the project code base.", required=True)
-    required_params.add_argument("-t", action="store", dest="project_type", choices=["maven", "npm", "nuget"], help="System managing the project external dependencies.", required=True)
+    required_params.add_argument("-t", action="store", dest="project_type", choices=["maven", "npm", "nuget", "pip"], help="System managing the project external dependencies.", required=True)
     parser.add_argument("-e", action="store", dest="gids_to_ignore", help="[MAVEN ONLY] List of artefacts GroupID, separated by a comma, to ignores (excludes) from the final POM file.", required=False, default="")
     args = parser.parse_args()
     project_descriptor_file_name = None
@@ -338,6 +391,9 @@ def is_npm_package_json_file(file_content):
     if args.project_type == "nuget":
         project_descriptor_file_name = "*.csproj"
         gids_to_ignore = None
+    if args.project_type == "pip":
+        project_descriptor_file_name = "pyproject.toml"
+        gids_to_ignore = None
     print(colored(f"[+] Extract all dependencies from all '{project_descriptor_file_name}' files...", "yellow"))
     global_dependencies = []
     global_properties = {}
@@ -357,6 +413,9 @@ def is_npm_package_json_file(file_content):
         if args.project_type == "nuget":
             dependencies = list_packages_nuget(content)
             global_dependencies.extend(dependencies)
+        if args.project_type == "pip":
+            dependencies = list_packages_pip(content)
+            global_dependencies.extend(dependencies)
     global_dependencies = list(set(global_dependencies))
     print(f"\rDependencies identified ({project_descriptor_file_name_file_count} files read): {len(global_dependencies):<80}")
     print(colored(f"[+] Identify all resolvable dependencies...", "yellow"))