Capture maintainers of packages too

Author: sethmlarson

.dockerfile

@@ -0,0 +1,3 @@
+venv
+*.db
+

Dockerfile

@@ -0,0 +1,6 @@
+FROM python:3.9-slim
+
+WORKDIR /pypi-data
+COPY . /pypi-data
+RUN python -m pip install -r requirements.txt
+CMD python main.py

README.md

@@ -5,8 +5,7 @@ Mostly up-to-date data about almost every package on PyPI
 Get access to the database via [GitHub releases](https://github.com/sethmlarson/pypi-data/releases).
 
 ```console
-$ git clone https://github.com/sethmlarson/pypi-data
-$ cd pypi-data
+$ gunzip pypi.db.gz
 $ sqlite3 'pypi.db' 'SELECT * FROM packages LIMIT 10 OFFSET 1000;'
 
 acid-vault|1.3.2|>=3.6|1|0|2021-01-21 04:37:10
@@ -29,6 +28,7 @@ ackg|0.0.5||0|0|2021-01-21 04:37:10
 - Python requirements (`requires_python`)
 - Yanked versions (`yanked`)
 - Wheel data (`python_tags`, `abi_tags`, `platform_tags`)
+- Maintainers on PyPI
 
 ### Database Schemas
 
@@ -64,6 +64,19 @@ CREATE TABLE wheels (
     abi STRING,
     platform STRING
 );
+
+-- Maintainer data --
+CREATE TABLE wheels (
+    name STRING,
+    package_name STRING
+);
+```
+
+## Running locally
+
+```
+$ docker build -t pypi-data .
+$ docker run --rm pypi-data
 ```
 
 ## License

main.py

@@ -70,6 +70,12 @@
   );
 """
 )
+db.execute("""
+    CREATE TABLE IF NOT EXISTS maintainers (
+        name STRING,
+        package_name STRING
+    );
+""")
 db.commit()
 
 
@@ -179,10 +185,20 @@ def get_metadata_by_install(package, resp):
     resp = resp.copy()
     resp["info"]["requires_dist"] = package_metadata["requires_dist"]
     resp["info"]["requires_python"] = package_metadata["requires_python"]
-    print(package_metadata)
     return resp
 
 
+def get_maintainers_from_pypi(package: str):
+    for _ in range(5):
+        resp = http.request("GET", f"https://pypi.org/project/{package}")
+        if resp.status == 404:
+            return set()
+        elif resp.status != 200:
+            continue
+        return set(re.findall(r"<a href=\"/user/([^/]+)/\" aria-label=", resp.data.decode("utf-8")))
+    return set()
+
+
 def update_data_from_pypi():
     for package in tqdm(packages, unit="packages"):
         resp = http.request("GET", f"https://pypi.org/pypi/{package}/json")
@@ -289,6 +305,11 @@ def update_data_from_pypi():
         )
         db.commit()
 
+        for maintainer in get_maintainers_from_pypi(package):
+            db.execute("""
+                INSERT OR IGNORE INTO maintainers (name, package_name) VALUES (?, ?);
+            """, (maintainer, package))
+
         for req in urequires_dist:
             extras = get_extras(req)
             req_no_specifiers = dist_from_requires_dist(req)
@@ -341,5 +362,4 @@ def update_data_from_pypi():
 
         db.commit()
 
-
 update_data_from_pypi()