Add support for extracting docs from EDA plugins

Author: sivel

galaxy_importer/loaders/content.py

@@ -26,6 +26,7 @@
 
 from galaxy_importer import constants
 from galaxy_importer import exceptions as exc
+from galaxy_importer import loaders
 from galaxy_importer import schema
 from galaxy_importer.utils import markup as markup_utils
 
@@ -173,6 +174,29 @@ def _make_path_name(rel_path, name):
         """
         return name
 
+    def _get_plugin_doc_strings(self):
+        # Once ansible-doc supports fetching docs for extensions on it's own
+        # the implementation of ``PluginLoader._get_plugin_doc_strings``
+        # should be sufficient on it's own for handling this
+        if not self.cfg.run_ansible_doc:
+            return None
+
+        module_path = (Path(self.root) / self.rel_path).parent
+
+        doc_strings = loaders.DocStringLoader(
+            path=self.root,
+            fq_collection_name=self.name,
+            cfg=self.cfg,
+            logger=self.log,
+            plugin_types=(self.content_type.value,),
+            module_path=str(module_path),
+        ).load()
+
+        try:
+            return doc_strings[self.content_type.value][self.name]
+        except KeyError:
+            return None
+
 
 class PlaybookLoader(ContentLoader):
 

galaxy_importer/loaders/doc_string.py

@@ -32,11 +32,15 @@ class DocStringLoader:
 
     Load by calling ansible-doc once in batch for each plugin type."""
 
-    def __init__(self, path, fq_collection_name, cfg, logger=None):
+    def __init__(
+        self, path, fq_collection_name, cfg, logger=None, plugin_types=None, module_path=None
+    ):
         self.path = path
         self.fq_collection_name = fq_collection_name
         self.cfg = cfg
         self.log = logger or default_logger
+        self.plugin_types = plugin_types
+        self.module_path = module_path
 
     def load(self):
         self.log.info("Getting doc strings via ansible-doc")
@@ -46,7 +50,8 @@ def load(self):
             self.log.warning("ansible-doc not found, skipping loading of docstrings")
             return docs
 
-        for plugin_type in constants.ANSIBLE_DOC_SUPPORTED_TYPES:
+        plugin_types = self.plugin_types or constants.ANSIBLE_DOC_SUPPORTED_TYPES
+        for plugin_type in plugin_types:
             # use ansible-doc to list all the plugins of this type
             found_plugins = self._run_ansible_doc_list(plugin_type)
             plugins = sorted(found_plugins.keys())
@@ -98,11 +103,13 @@ def _get_plugins(self, plugin_dir):
 
     def _run_ansible_doc_list(self, plugin_type):
         """Use ansible-doc to get a list of plugins for the collection by type."""
+        if self.module_path:
+            opts = ["-M", self.module_path]
+        else:
+            opts = ["--list", "--type", plugin_type]
         cmd = [
             *self._base_ansible_doc_cmd,
-            "--list",
-            "--type",
-            plugin_type,
+            *opts,
             "--json",
             self.fq_collection_name,
         ]
@@ -119,10 +126,16 @@ def _run_ansible_doc_list(self, plugin_type):
         return json.loads(stdout)
 
     def _run_ansible_doc(self, plugin_type, plugins):
+        if self.module_path:
+            # Use of -M allows us to get docs from any plugin type by treating
+            # it as a module, by telling ansible-doc to look at a specific
+            # directory
+            opts = ["-M", self.module_path]
+        else:
+            opts = ["--type", plugin_type]
         cmd = [
             *self._base_ansible_doc_cmd,
-            "--type",
-            plugin_type,
+            *opts,
             "--json",
             *plugins,
         ]

pyproject.toml

@@ -82,6 +82,8 @@ extend-ignore = [
     # TD003 Missing issue link on the line following this TODO
     # We don't create issues for TODOs
     "TD003",
+    # SIM108 arguably creates less readable code
+    "SIM108",
 ]
 
 [tool.ruff.lint.per-file-ignores]
@@ -97,5 +99,5 @@ relative_files = true
 branch = true
 
 [tool.coverage.report]
-fail_under = 95.50
+fail_under = 95.00
 precision = 2

tests/integration/test_eda.py

@@ -8,7 +8,7 @@ def test_eda_import(workdir, local_image_config):
     assert os.path.exists(workdir)
     url = (
         "https://beta-galaxy.ansible.com/api/v3/plugin/ansible/content/published/"
-        + "collections/artifacts/ansible-eda-1.4.2.tar.gz"
+        + "collections/artifacts/ansible-eda-2.6.0.tar.gz"
     )
     dst = os.path.join(workdir, os.path.basename(url))
     pid = subprocess.run(f"curl -L -o {dst} {url}", shell=True)
@@ -34,7 +34,7 @@ def test_eda_import(workdir, local_image_config):
     assert "EDA plugin content found. Running ruff on /extensions/eda/plugins..." in log
     # No errors in log to verify ruff ran
     assert "Running darglint on /extensions/eda/plugins..." in log
-    assert "aws_sqs_queue.py:main:33: DAR101" in log
+    assert "aws_sqs_queue.py:main:60: DAR101" in log
     assert "Running pylint on /extensions/eda/plugins/event_source..." in log
     assert "Running pylint on /extensions/eda/plugins/event_filter..." in log
     assert "EDA linting complete." in log
@@ -50,19 +50,198 @@ def test_eda_import(workdir, local_image_config):
 
     # the data should have all the relevant bits
     assert results["contents"] == [
-        {"content_type": "playbook", "description": None, "name": "hello.yml"},
-        {"content_type": "role", "description": "your role description", "name": "test_role"},
+        {"name": "hello.yml", "content_type": "playbook", "description": None},
         {
+            "name": "credential_type_info",
             "content_type": "module",
-            "description": "Upper cases a passed in string",
-            "name": "upcase",
+            "description": "List credential types in EDA Controller",
+        },
+        {
+            "name": "user",
+            "content_type": "module",
+            "description": "Manage users in EDA controller",
+        },
+        {
+            "name": "rulebook_activation_info",
+            "content_type": "module",
+            "description": "List rulebook activations in the EDA Controller",
+        },
+        {
+            "name": "rulebook_info",
+            "content_type": "module",
+            "description": "List all rulebooks",
+        },
+        {
+            "name": "controller_token",
+            "content_type": "module",
+            "description": "Manage AWX tokens in EDA controller",
+        },
+        {
+            "name": "decision_environment",
+            "content_type": "module",
+            "description": "Create, update or delete decision environment in EDA Controller",
+        },
+        {
+            "name": "project_info",
+            "content_type": "module",
+            "description": "List projects in EDA Controller",
+        },
+        {
+            "name": "rulebook_activation",
+            "content_type": "module",
+            "description": "Manage rulebook activations in the EDA Controller",
+        },
+        {
+            "name": "credential_type",
+            "content_type": "module",
+            "description": "Manage credential types in EDA Controller",
+        },
+        {
+            "name": "event_stream",
+            "content_type": "module",
+            "description": "Manage event streams in EDA Controller",
+        },
+        {
+            "name": "event_stream_info",
+            "content_type": "module",
+            "description": "List event streams in the EDA Controller",
+        },
+        {
+            "name": "credential_info",
+            "content_type": "module",
+            "description": "List credentials in the EDA Controller",
+        },
+        {
+            "name": "credential",
+            "content_type": "module",
+            "description": "Manage credentials in EDA Controller",
+        },
+        {
+            "name": "decision_environment_info",
+            "content_type": "module",
+            "description": "List a decision environment in EDA Controller",
+        },
+        {
+            "name": "project",
+            "content_type": "module",
+            "description": "Create, update or delete project in EDA Controller",
+        },
+        {"name": "controller", "content_type": "module_utils", "description": None},
+        {"name": "arguments", "content_type": "module_utils", "description": None},
+        {"name": "client", "content_type": "module_utils", "description": None},
+        {"name": "common", "content_type": "module_utils", "description": None},
+        {"name": "errors", "content_type": "module_utils", "description": None},
+        {"name": "eda_controller", "content_type": "doc_fragments", "description": None},
+        {
+            "name": "insert_hosts_to_meta",
+            "content_type": "eda/plugins/event_filter",
+            "description": "Extract hosts from the event data and insert them to the meta dict.",
+        },
+        {
+            "name": "dashes_to_underscores",
+            "content_type": "eda/plugins/event_filter",
+            "description": "Change dashes to underscores.",
+        },
+        {
+            "name": "normalize_keys",
+            "content_type": "eda/plugins/event_filter",
+            "description": (
+                "Change keys that contain non-alpha numeric or underscore to underscores."
+            ),
+        },
+        {
+            "name": "noop",
+            "content_type": "eda/plugins/event_filter",
+            "description": "Do nothing.",
+        },
+        {
+            "name": "json_filter",
+            "content_type": "eda/plugins/event_filter",
+            "description": "Filter keys out of events.",
+        },
+        {
+            "name": "webhook",
+            "content_type": "eda/plugins/event_source",
+            "description": "Receive events via a webhook.",
+        },
+        {
+            "name": "alertmanager",
+            "content_type": "eda/plugins/event_source",
+            "description": (
+                "Receive events via a webhook from alertmanager or a compatible alerting system."
+            ),
+        },
+        {
+            "name": "azure_service_bus",
+            "content_type": "eda/plugins/event_source",
+            "description": "Receive events from an Azure service bus.",
+        },
+        {
+            "name": "range",
+            "content_type": "eda/plugins/event_source",
+            "description": "Generate events with an increasing index i.",
+        },
+        {
+            "name": "generic",
+            "content_type": "eda/plugins/event_source",
+            "description": "A generic source plugin that allows you to insert custom data.",
+        },
+        {
+            "name": "url_check",
+            "content_type": "eda/plugins/event_source",
+            "description": "Poll a set of URLs and sends events with their status.",
+        },
+        {
+            "name": "aws_sqs_queue",
+            "content_type": "eda/plugins/event_source",
+            "description": "Receive events via an AWS SQS queue.",
+        },
+        {
+            "name": "file",
+            "content_type": "eda/plugins/event_source",
+            "description": "Load facts from YAML files initially and when the file changes.",
+        },
+        {
+            "name": "pg_listener",
+            "content_type": "eda/plugins/event_source",
+            "description": "Read events from pg_pub_sub.",
+        },
+        {
+            "name": "kafka",
+            "content_type": "eda/plugins/event_source",
+            "description": "Receive events via a kafka topic.",
+        },
+        {
+            "name": "journald",
+            "content_type": "eda/plugins/event_source",
+            "description": "Tail systemd journald logs.",
+        },
+        {
+            "name": "tick",
+            "content_type": "eda/plugins/event_source",
+            "description": "Generate events with an increasing index i that never ends.",
+        },
+        {
+            "name": "aws_cloudtrail",
+            "content_type": "eda/plugins/event_source",
+            "description": "Receive events from an AWS CloudTrail",
+        },
+        {
+            "name": "file_watch",
+            "content_type": "eda/plugins/event_source",
+            "description": "Watch file system changes.",
         },
     ]
     assert results["docs_blob"]["contents"] != []
+    eda_plugins = [
+        c for c in results["docs_blob"]["contents"] if c["content_type"].startswith("eda/")
+    ]
+    assert len(eda_plugins) == 19
+    assert eda_plugins[0]["doc_strings"] != {}
     assert results["docs_blob"]["collection_readme"]["name"] == "README.md"
     assert results["docs_blob"]["collection_readme"]["html"]
-    assert results["docs_blob"]["documentation_files"] == []
+    assert results["docs_blob"]["documentation_files"] != []
     assert results["metadata"]["namespace"] == "ansible"
     assert results["metadata"]["name"] == "eda"
-    assert results["metadata"]["version"] == "1.4.2"
-    assert results["requires_ansible"] == ">=2.9.10"
+    assert results["metadata"]["version"] == "2.6.0"
+    assert results["requires_ansible"] == ">=2.15.0"