Add extra data to GH section (#980)

Add `meta` in `ecosystem.json`

Extend `ecosystem.json` with extra fields:

Deprecate some `ecosystem.json` fields:
```
        "members.created_at": "currently not in use. To be removed in v2.",
        "members.updated_at": "currently not in use. To be removed in v2.",
        "members.contact_info": "currently not in use. To be removed in v2.",
        "members.affiliations": "currently not in use. To be removed in v2.",
        "members.group": "replaced by members.category. To be removed in v2.",
        "members.stars": "replaced by members.github.stars. To be removed in v2.",
        "members.url": "replaced by members.github.url. To be removed in v2.",
        "members.website": "replaced by members.website.home. To be removed in v2.",
```

Support for search in json

Add `github.total_dependent_repositories` and
`github.total_dependent_packages`.

Fetch License and description from github

Author: 1ucian0

ecosystem/cli/members.py

@@ -88,7 +88,6 @@ def update_github(self):
         """Updates GitHub data."""
         for project in self.dao.get_all():
             project.update_github()
-            self.dao.update(project.name_id, stars=project.github.stars)
             self.dao.update(project.name_id, github=project.github)
 
     def update_pypi(self):
@@ -98,24 +97,36 @@ def update_pypi(self):
             self.dao.update(project.name_id, pypi=project.pypi)
 
     @staticmethod
-    def filter_member_data(member_dict, data_map):
+    def filter_member_data(
+        member_dict, data_map, forced_addition=False
+    ):  # pylint: disable=too-many-branches
         """takes a member dictionary and a data map,
-        and returns a dict that is filtered by the map"""
+        and returns a dict that is filtered by the map.
+        If forced_addition is True, all the elements of the
+        data_map will be added, even if they are empty"""
         filtered_data = {}
         for key, alias in data_map.items():
             if isinstance(alias, dict):
                 data = CliMembers.filter_member_data(member_dict, alias)
                 if data:
                     filtered_data[key] = data
-            elif isinstance(alias, list):
+            elif isinstance(alias, tuple):
                 if len(alias) != 2:
                     raise ValueError(
                         "%s malformed. "
                         "It needs to have exactly two elements,one "
                         "with the query, the otherone with the selector"
                     )
                 data = list(query(alias[0], member_dict).select(*alias[1]))
-
+            elif isinstance(alias, list):
+                # a list of alias in priority in case they do not exist
+                for candidate_alias in alias:
+                    candidate_value = CliMembers.filter_member_data(
+                        member_dict, {key: candidate_alias}
+                    )
+                    if len(candidate_value) == 1:
+                        data = list(candidate_value.values())[0]
+                        break
             else:
                 found_all = findall(alias, member_dict)
                 if len(found_all) == 0:
@@ -127,7 +138,7 @@ def filter_member_data(member_dict, data_map):
                         f"I dont know who to hangle multiple results for {found_all}. "
                         "Maybe functools.reduce?"
                     )
-            if data:
+            if forced_addition or data:
                 filtered_data[key] = data
         return filtered_data
 
@@ -137,41 +148,21 @@ def compile_json(self, output_file: str):
             "uuid": "uuid",
             "name": "name",
             "url": "github.url",
-            "description": "description",
-            "licence": "licence",
+            "description": ["description", "github.description"],
+            "licence": ["licence", "github.license"],
             "contact_info": "contact_info",
             "affiliations": "affiliations",
             "labels": "labels",
             "created_at": "created_at",
             "updated_at": "updated_at",
-            "group": "group",
+            "group": "category",
+            "category": "category",
             "stars": "github.stars",
             "documentation": "documentation",
             "website": "website",
             "reference_paper": "reference_paper",
-            "ibm_maintained": "ibm_maintained",
-        }
-        data = {
-            "members": [
-                CliMembers.filter_member_data(member.to_dict(), member_data_to_export)
-                for member in self.dao.get_all()
-            ],
-            "labels": json.loads(Path(self.resources_dir, "labels.json").read_text()),
-        }
-        Path(output_file).write_text(
-            json.dumps(data, default=str, separators=(",", ":"))
-        )
-
-    def compile_json_v1(self, output_file: str):
-        """Compile JSON file (v1) for consumption by ibm.com"""
-        member_data_to_export = {
-            "name": "name",
-            "description": "description",
-            "licence": "licence",
-            "subjects": "labels",
-            "type": "group",
+            "ibm_maintained": "ibm_maintained",  # if _prefixed, then it adding it is forced
             "badge": "badge",
-            "ibm_maintained": "ibm_maintained",
             "websites": {
                 "home": "website",
                 "documentation": "documentation",
@@ -183,7 +174,7 @@ def compile_json_v1(self, output_file: str):
                 "last_commit": "github.last_commit",
                 "archived": "github.archived",
             },
-            "python_packages": [
+            "python_packages": (
                 "pypi.*",
                 [
                     "package_name",
@@ -192,11 +183,28 @@ def compile_json_v1(self, output_file: str):
                     "compatible_with_qiskit_v1",
                     "compatible_with_qiskit_v2",
                 ],
-            ],
+            ),
         }
         data = {
+            "meta": {
+                "version": 1,
+                "deprecated fields": {
+                    "members.created_at": "currently not in use. To be removed in v2.",
+                    "members.updated_at": "currently not in use. To be removed in v2.",
+                    "members.contact_info": "currently not in use. To be removed in v2.",
+                    "members.affiliations": "currently not in use. To be removed in v2.",
+                    "members.group": "replaced by members.category. To be removed in v2.",
+                    "members.stars": "replaced by members.github.stars. To be removed in v2.",
+                    "members.url": "replaced by members.github.url. To be removed in v2.",
+                    "members.website": "replaced by members.website.home. To be removed in v2.",
+                    "reference_paper": "replaced by members.website.reference_paper. "
+                    "To be removed in v2.",
+                },
+            },
             "members": [
-                CliMembers.filter_member_data(member.to_dict(), member_data_to_export)
+                CliMembers.filter_member_data(
+                    member.to_dict(), member_data_to_export, forced_addition=True
+                )
                 for member in self.dao.get_all()
             ],
             "labels": json.loads(Path(self.resources_dir, "labels.json").read_text()),

ecosystem/github.py

@@ -7,7 +7,7 @@
 
 from .serializable import JsonSerializable, parse_datetime
 from .error_handling import EcosystemError, logger
-from .request import request_json
+from .request import request_json, parse_github_package_ids, parse_github_dependants
 
 
 class GitHubData(JsonSerializable):
@@ -21,18 +21,27 @@ class GitHubData(JsonSerializable):
         "repo",
         "tree",
         "stars",
+        "homepage",
+        "license",
+        "description",
+        "total_dependent_repositories",
+        "total_dependent_packages",
         "private",
         "archived",
+        "disabled",
         "last_commit",
     ]
     aliases = {
         "stars": "stargazers_count",
         "last_commit": "pushed_at",
         "url": "html_url",
+        "license": "license.name",
     }
     json_types = {
+        "homepage": lambda x: x or None,
         "private": lambda x: x or None,
         "archived": lambda x: x or None,
+        "disabled": lambda x: x or None,
         "pushed_at": parse_datetime,
     }
     reduce = {}
@@ -42,7 +51,9 @@ def __init__(self, owner: str, repo: str, tree: str = None, **kwargs):
         self.repo = repo
         self.tree = tree
         self._kwargs = kwargs or {}
-        self._json_data = None
+        self._json_repo = None
+        self._json_package_ids = None
+        self._json_dependants = None
 
     def to_dict(self) -> dict:
         dictionary = {}
@@ -84,16 +95,29 @@ def from_url(cls, github_project_url: ParseResult):
 
     def update_json(self):
         """
-        Fetches remote json data from api.github.com/repos/{self.owner}/{self.repo}
+        Fetches remote data from:
+          - api.github.com/repos/{self.owner}/{self.repo}
+          - github.com/{self.owner}/{self.repo}/network/dependents
         """
-        self._json_data = request_json(f"api.github.com/repos/{self.owner}/{self.repo}")
+        self._json_repo = request_json(f"api.github.com/repos/{self.owner}/{self.repo}")
+        self._json_package_ids = request_json(
+            f"github.com/{self.owner}/{self.repo}/network/dependents?dependent_type=REPOSITORY",
+            parser=parse_github_package_ids,
+        )
+        self._json_dependants = {}
+        for package, package_id in self._json_package_ids.items():
+            self._json_dependants[package] = request_json(
+                f"github.com/{self.owner}/{self.repo}/network/dependents?"
+                f"dependent_type=REPOSITORY&package_id={package_id}",
+                parser=parse_github_dependants,
+            )
 
     def __getattr__(self, item):
-        if self._json_data:
+        if self._json_repo:
             if item in GitHubData.aliases:
                 item = GitHubData.aliases[item]
 
-            json_elements = findall(item, self._json_data)
+            json_elements = findall(item, self._json_repo)
             if item in GitHubData.json_types:
                 json_elements = [GitHubData.json_types[item](e) for e in json_elements]
 
@@ -118,11 +142,31 @@ def update_owner_repo(self):
         """
         Updates GitHub page when the repo was moved or renamed
         """
-        if self._json_data is None:
+        if self._json_repo is None:
             self.update_json()
-        owner = self._json_data["owner"]["login"]
-        repo = self._json_data["name"]
+        owner = self._json_repo["owner"]["login"]
+        repo = self._json_repo["name"]
         if self.owner != owner or self.repo != repo:
             logger.info("%s/%s moved to %s/%s", self.owner, self.repo, owner, repo)
             self.owner = owner
             self.repo = repo
+
+    def dependants(self, refresh=False):
+        """get the dependant data from (cached) JSON"""
+        if refresh:
+            self.update_json()
+        return self._json_dependants
+
+    @property
+    def total_dependent_repositories(self):
+        """Sum of repository dependants"""
+        if self.dependants():
+            return sum(r.get("repositories", 0) for r in self.dependants().values())
+        return None
+
+    @property
+    def total_dependent_packages(self):
+        """Sum of package dependants"""
+        if self.dependants():
+            return sum(r.get("packages", 0) for r in self.dependants().values())
+        return None

ecosystem/member.py

@@ -26,8 +26,8 @@ class Member(JsonSerializable):
     created_at: int | None = None
     updated_at: int | None = None
     website: str | None = None
-    stars: int | None = None
     group: str | None = None
+    category: str | None = None
     reference_paper: str | None = None
     documentation: str | None = None
     packages: list[str] | None = None
@@ -80,6 +80,8 @@ def to_dict(self) -> dict:
             f"%3A%2F%2Fqiskit.github.io%2Fecosystem%2Fb%2F{self.short_uuid})]"
             "(https://qisk.it/e)"
         )
+        if "ibm_maintained" in base_dict and base_dict["ibm_maintained"] is False:
+            del base_dict["ibm_maintained"]
         return base_dict
 
     def __eq__(self, other: "Member"):