Integration Secrets Manager in Perceval

Signed-off-by: Alberto Ferrer Sánchez <alberefe@gmail.com>

Author: alberefe

README.md

@@ -143,6 +143,132 @@ A Perceval Docker image is available at
 Detailed information on how to run and/or build this image can be found
 [here](https://github.com/chaoss/grimoirelab-perceval/tree/main/docker/images/).
 
+## Secrets Manager
+
+Perceval supports retrieving credentials from a secrets manager instead of
+passing them directly on the command line. This is useful for automated
+pipelines and environments where storing credentials in plain text is not
+acceptable.
+
+The following backends support secrets manager authentication: `bugzilla`,
+`bugzillarest`, `confluence`, `discourse`, `gerrit`, `git`, `github`,
+`gitlab`, `gitter`, `googlehits`, `groupsio`, `jenkins`, `rocketchat`,
+`stackexchange`.
+
+### Installation
+
+To use **HashiCorp Vault**, install Perceval with the `hashicorp-manager` group:
+
+```
+$ poetry install --with hashicorp-manager
+```
+
+**Bitwarden** does not require any extra Python package, but the
+[Bitwarden CLI](https://bitwarden.com/help/cli/) (`bw`) must be installed
+and available on your `PATH`.
+
+### Common arguments
+
+All secrets manager providers share the following arguments:
+
+- `--secrets-manager` — Provider to use: `bitwarden` or `hashicorp`
+- `--item-name` — Name of the secret item in the secrets manager
+
+Credentials must be stored in the vault using the same field names that
+Perceval expects: `user`, `password`, `api_token`, `email`, `access_token`,
+`user_id`. Perceval automatically looks up all of these fields and uses
+whichever ones are present.
+
+### Expected field names
+
+Store your credentials in the vault using these exact names:
+
+- `api_token` — used by `github`, `gitlab`, `bugzillarest`, `stackexchange`, `gitter`
+- `user` — used by `bugzilla`, `confluence`, `discourse`, `gerrit`, `jenkins`
+- `password` — used by `bugzilla`, `confluence`, `discourse`, `gerrit`, `jenkins`
+- `email` — used by `groupsio`
+- `access_token` — used by `groupsio`
+- `user_id` — used by `rocketchat`
+
+Only the fields present in the vault are used; the rest are silently ignored.
+
+### Bitwarden
+
+Store your credentials in a Bitwarden item using Perceval's expected field
+names. Fields can be stored as login fields or as custom fields.
+
+For example, to store a GitHub API token, create a Bitwarden item named
+`GitHub` with a custom field named `api_token` containing the token value.
+
+Bitwarden-specific arguments:
+
+- `--bw-client-id` — Bitwarden API client ID
+- `--bw-client-secret` — Bitwarden API client secret
+- `--bw-master-password` — Bitwarden master password
+
+#### Example
+```
+$ perceval github \
+    --secrets-manager bitwarden \
+    --item-name 'GitHub' \
+    --bw-client-id $BW_CLIENT_ID \
+    --bw-client-secret $BW_CLIENT_SECRET \
+    --bw-master-password $BW_MASTER_PASSWORD \
+    --from-date '2020-01-01' --no-archive \
+    chaoss grimoirelab-perceval
+```
+
+### HashiCorp Vault
+
+Store your credentials as key-value pairs in a HashiCorp Vault KV secret
+using Perceval's expected field names.
+
+For example, to store a GitHub API token:
+```
+$ vault kv put secret/GitHub api_token=ghp_xxxxxxxxxxxx
+```
+
+HashiCorp-specific arguments:
+
+- `--vault-url` — HashiCorp Vault server URL
+- `--vault-token` — Vault authentication token
+- `--vault-certificate` — Path to CA certificate for TLS verification (optional)
+
+#### Example
+```
+$ perceval github \
+    --secrets-manager hashicorp \
+    --item-name 'GitHub' \
+    --vault-url $VAULT_URL \
+    --vault-token $VAULT_TOKEN \
+    --from-date '2020-01-01' --no-archive \
+    chaoss grimoirelab-perceval
+```
+
+### Programmatic usage
+
+The credential resolution logic is also available in `grimoirelab-toolkit`
+via the `CredentialManager` base class, so it can be used from any Python
+code without going through the CLI:
+
+```python
+from grimoirelab_toolkit.credential_manager import BitwardenManager
+from grimoirelab_toolkit.credential_manager.hc_manager import HashicorpManager
+
+# Bitwarden example
+bw_manager = BitwardenManager("your-client-id", "your-client-secret", "your-master-password")
+credentials = bw_manager.resolve_credentials("GitHub", ["api_token"])
+print(credentials)  # {'api_token': 'ghp_...'}
+
+# HashiCorp Vault example
+hc_manager = HashicorpManager("https://vault.example.com", "hvs.your-token")
+credentials = hc_manager.resolve_credentials("secret/GitHub", ["api_token"])
+print(credentials)  # {'api_token': 'ghp_...'}
+```
+
+This is useful for consumers like KingArthur or custom scripts that use
+Perceval's `Backend` class directly without the CLI.
+
 ## Documentation
 
 Documentation is generated automatically in the [ReadTheDocs Perceval

perceval/backend.py

@@ -617,12 +617,14 @@ class BackendCommandArgumentParser:
 
     def __init__(self, backend, from_date=False, to_date=False, offset=False,
                  basic_auth=False, token_auth=False, archive=False,
-                 aliases=None, blacklist=False, ssl_verify=False):
+                 aliases=None, blacklist=False, ssl_verify=False,
+                 secrets_manager=False):
         self._from_date = from_date
         self._to_date = to_date
         self._archive = archive
         self._backend = backend
         self._ssl_verify = ssl_verify
+        self._secrets_manager = secrets_manager
 
         self.aliases = aliases or {}
         self.parser = argparse.ArgumentParser()
@@ -673,6 +675,9 @@ def __init__(self, backend, from_date=False, to_date=False, offset=False,
             group.add_argument('--no-ssl-verify', dest='ssl_verify', action='store_false',
                                help="disable SSL verification")
 
+        if secrets_manager:
+            self._set_secrets_manager_arguments()
+
         self._set_output_arguments()
 
     def parse(self, *args):
@@ -749,6 +754,38 @@ def _set_output_arguments(self):
         group.add_argument('--json-line', dest='json_line', action='store_true',
                            help="produce a JSON line for each output item")
 
+    def _set_secrets_manager_arguments(self):
+        """Activate secret manager arguments parsing"""
+
+        group = self.parser.add_argument_group('secrets manager arguments')
+        group.add_argument('--secrets-manager', dest='secrets_manager',
+                           choices=['bitwarden', 'hashicorp'],
+                           help="Secrets manager service to use for credential retrieval")
+        group.add_argument('--item-name', dest='item_name',
+                           help="Name of the item in the secrets manager")
+
+        bw_group = self.parser.add_argument_group('bitwarden arguments')
+        bw_group.add_argument('--bw-client-id', dest='bw_client_id',
+                              default=os.environ.get('PERCEVAL_BW_CLIENT_ID'),
+                              help="Bitwarden API client ID (env: PERCEVAL_BW_CLIENT_ID)")
+        bw_group.add_argument('--bw-client-secret', dest='bw_client_secret',
+                              default=os.environ.get('PERCEVAL_BW_CLIENT_SECRET'),
+                              help="Bitwarden API client secret (env: PERCEVAL_BW_CLIENT_SECRET)")
+        bw_group.add_argument('--bw-master-password', dest='bw_master_password',
+                              default=os.environ.get('PERCEVAL_BW_MASTER_PASSWORD'),
+                              help="Bitwarden master password (env: PERCEVAL_BW_MASTER_PASSWORD)")
+
+        hc_group = self.parser.add_argument_group('hashicorp vault arguments')
+        hc_group.add_argument('--vault-url', dest='vault_url',
+                              default=os.environ.get('PERCEVAL_VAULT_URL'),
+                              help="HashiCorp Vault URL (env: PERCEVAL_VAULT_URL)")
+        hc_group.add_argument('--vault-token', dest='vault_token',
+                              default=os.environ.get('PERCEVAL_VAULT_TOKEN'),
+                              help="HashiCorp Vault authentication token (env: PERCEVAL_VAULT_TOKEN)")
+        hc_group.add_argument('--vault-certificate', dest='vault_certificate',
+                              default=os.environ.get('PERCEVAL_VAULT_CERTIFICATE'),
+                              help="Path to CA certificate for HashiCorp Vault TLS verification (env: PERCEVAL_VAULT_CERTIFICATE)")
+
 
 class BackendCommand:
     """Abstract class to run backends from the command line.
@@ -822,8 +859,76 @@ def run(self):
                 logger.exception(f"Error!: {e}", exc_info=self.debug)
 
     def _pre_init(self):
-        """Override to execute before backend is initialized."""
-        pass
+        """Override to execute before backend is initialized.
+
+        This method handles fetching credentials from a secrets manager
+        and injecting them into backend arguments.
+        """
+        if not (hasattr(self.parsed_args, 'secrets_manager') and
+                self.parsed_args.secrets_manager):
+            return
+
+        if not getattr(self.parsed_args, 'item_name', None):
+            raise ValueError("--item-name is required when --secrets-manager is specified.")
+
+        logging.debug("Processing credentials with %s", self.parsed_args.secrets_manager)
+
+        try:
+            manager = self._build_manager()
+            field_names = ['user', 'password', 'api_token', 'email', 'access_token', 'user_id']
+
+            credentials = manager.resolve_credentials(
+                secret_name=self.parsed_args.item_name,
+                field_names=field_names,
+            )
+
+            # Post-process: GitHub backend expects api_token as a list
+            if 'api_token' in credentials:
+                credentials['api_token'] = [credentials['api_token']]
+
+            # Inject resolved credentials into parsed_args
+            for param_name, value in credentials.items():
+                setattr(self.parsed_args, param_name, value)
+                logger.info('Using %s from secrets manager', param_name)
+
+        except ImportError:
+            logging.warning('Credential management module not found. Using command line credentials.')
+        except Exception as e:
+            raise RuntimeError('Error retrieving credentials from secret manager: %s' % str(e)) from e
+
+    def _build_manager(self):
+        """Build and return a credential manager instance from parsed CLI args.
+
+        :returns: A credential manager instance
+        :rtype: CredentialManager
+        """
+        manager_type = self.parsed_args.secrets_manager
+
+        if manager_type == 'bitwarden':
+            bw_client_id = getattr(self.parsed_args, 'bw_client_id', None)
+            bw_client_secret = getattr(self.parsed_args, 'bw_client_secret', None)
+            bw_master_password = getattr(self.parsed_args, 'bw_master_password', None)
+
+            if not all([bw_client_id, bw_client_secret, bw_master_password]):
+                raise ValueError(
+                    'Bitwarden requires --bw-client-id, --bw-client-secret, and --bw-master-password'
+                )
+
+            from grimoirelab_toolkit.credential_manager.bw_manager import BitwardenManager
+            return BitwardenManager(bw_client_id, bw_client_secret, bw_master_password)
+
+        elif manager_type == 'hashicorp':
+            vault_url = getattr(self.parsed_args, 'vault_url', None)
+            vault_token = getattr(self.parsed_args, 'vault_token', None)
+            vault_certificate = getattr(self.parsed_args, 'vault_certificate', None)
+
+            if not all([vault_url, vault_token]):
+                raise ValueError('HashiCorp Vault requires --vault-url and --vault-token')
+
+            from grimoirelab_toolkit.credential_manager.hc_manager import HashicorpManager
+            return HashicorpManager(vault_url, vault_token, vault_certificate)
+
+        raise ValueError(f"Unsupported secrets manager: '{manager_type}'")
 
     def _post_init(self):
         """Override to execute after backend is initialized."""

perceval/backends/core/bugzilla.py

@@ -363,7 +363,8 @@ def setup_cmd_parser(cls):
                                               from_date=True,
                                               basic_auth=True,
                                               archive=True,
-                                              ssl_verify=True)
+                                              ssl_verify=True,
+                                              secrets_manager=True)
 
         # Bugzilla options
         group = parser.parser.add_argument_group('Bugzilla arguments')

perceval/backends/core/bugzillarest.py

@@ -507,7 +507,8 @@ def setup_cmd_parser(cls):
                                               basic_auth=True,
                                               token_auth=True,
                                               archive=True,
-                                              ssl_verify=True)
+                                              ssl_verify=True,
+                                              secrets_manager=True)
 
         # BugzillaREST options
         group = parser.parser.add_argument_group('Bugzilla REST arguments')

perceval/backends/core/confluence.py

@@ -351,7 +351,8 @@ def setup_cmd_parser(cls):
                                               basic_auth=True,
                                               token_auth=True,
                                               archive=True,
-                                              ssl_verify=True)
+                                              ssl_verify=True,
+                                              secrets_manager=True)
 
         # Required arguments
         parser.parser.add_argument('url',

perceval/backends/core/discourse.py

@@ -436,7 +436,8 @@ def setup_cmd_parser(cls):
                                               from_date=True,
                                               token_auth=True,
                                               archive=True,
-                                              ssl_verify=True)
+                                              ssl_verify=True,
+                                              secrets_manager=True)
 
         # Required arguments
         parser.parser.add_argument('url',

perceval/backends/core/gerrit.py

@@ -514,7 +514,8 @@ def setup_cmd_parser(cls):
         parser = BackendCommandArgumentParser(cls.BACKEND,
                                               from_date=True,
                                               archive=True,
-                                              blacklist=True)
+                                              blacklist=True,
+                                              secrets_manager=True)
 
         # Gerrit options
         group = parser.parser.add_argument_group('Gerrit arguments')

perceval/backends/core/git.py

@@ -405,6 +405,9 @@ class GitCommand(BackendCommand):
     def _pre_init(self):
         """Initialize repositories directory path"""
 
+        # Fetch credentials from secrets manager if configured
+        super()._pre_init()
+
         if self.parsed_args.git_log:
             git_path = self.parsed_args.git_log
         elif self.parsed_args.git_path:
@@ -427,7 +430,8 @@ def setup_cmd_parser(cls):
         parser = BackendCommandArgumentParser(cls.BACKEND,
                                               from_date=True,
                                               to_date=True,
-                                              ssl_verify=True)
+                                              ssl_verify=True,
+                                              secrets_manager=True)
 
         # Optional arguments
         group = parser.parser.add_argument_group('Git arguments')

perceval/backends/core/github.py

@@ -1176,7 +1176,8 @@ def setup_cmd_parser(cls):
                                               to_date=True,
                                               token_auth=False,
                                               archive=True,
-                                              ssl_verify=True)
+                                              ssl_verify=True,
+                                              secrets_manager=True)
         # GitHub options
         group = parser.parser.add_argument_group('GitHub arguments')
         group.add_argument('--enterprise-url', dest='base_url',

perceval/backends/core/gitlab.py

@@ -756,7 +756,8 @@ def setup_cmd_parser(cls):
                                               token_auth=True,
                                               archive=True,
                                               blacklist=True,
-                                              ssl_verify=True)
+                                              ssl_verify=True,
+                                              secrets_manager=True)
 
         # GitLab options
         group = parser.parser.add_argument_group('gitlab arguments')