Fix refspec, url for cloning projects in reproducer

Zuul reproducer does only seem to support cloning projects from github,
while trying to clone projects from gerrit it uses a wrong url and
refspec. This change introduces two filters to get the right values for
url and refspec for the different sources (github and gerrit).

Author: cescgina

ansible.cfg

@@ -2,6 +2,7 @@
 action_plugins = ./plugins/action:~/plugins/action:/usr/share/ansible/plugins/action
 library = ./plugins/modules:~/plugins/modules:/usr/share/ansible/plugins/modules
 roles_path = ~/ci-framework-data/artifacts/roles:./roles:/usr/share/ansible/roles:/etc/ansible/roles:~/.ansible/roles
+filter_plugins = ./plugins/filter:~/plugins/filter:/usr/share/ansible/plugins/filter
 log_path = ~/ansible.log
 # We may consider ansible.builtin.junit
 callbacks_enabled = ansible.posix.profile_tasks

plugins/filter/reproducer_gerrit_infix.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+
+__metaclass__ = type
+
+DOCUMENTATION = """
+    name: reproducer_gerrit_infix
+    short_description: Maps a hostname to the infix needed for cloning a repo
+    for that hostname.
+    description:
+        - Maps a hostname to the infix needed for cloning a repo from that
+          hostname.Some gerrit instances add an infix like "gerrit", or "r"
+          to the repo clone url.
+    options:
+        _input:
+            description:
+                - repo hostname where we want to clone from.
+            type: str
+            required: true
+"""
+
+EXAMPLES = """
+    - name: Get infix to clone repo from rdo gerrit
+      ansible.builtin.set_fact:
+        gerrit_infix: >-
+            {{
+              "review.rdoproject.org" | reproducer_gerrit_infix
+            }}
+"""
+
+RETURN = """
+  _value:
+    description: The infix to add to the url to clone the repo.
+    type: str
+    sample:
+      "gerrit/"
+
+"""
+
+from ansible.errors import AnsibleFilterTypeError
+
+
+class FilterModule:
+    @classmethod
+    def __map_hostname_infix(cls, hostname):
+        if not isinstance(hostname, str):
+            raise AnsibleFilterTypeError(
+                f"reproducer_gerrit_infix requires a str, got {type(hostname)}"
+            )
+        if "rdoproject" in hostname:
+            return "r/"
+        elif "code.eng" in hostname:
+            return "gerrit/"
+        return ""
+
+    def filters(self):
+        return {
+            "reproducer_gerrit_infix": self.__map_hostname_infix,
+        }

plugins/filter/reproducer_refspec.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+__metaclass__ = type
+
+DOCUMENTATION = """
+    name: reproducer_refspec
+    short_description: Maps repo information from zuul to a refspec.
+    description:
+        - Maps repo information from zuul to a refspec that should be
+        used for pulling it with git.
+    options:
+        _input:
+            description:
+                - repo information from zuul variables.
+            type: dict
+            required: true
+"""
+
+EXAMPLES = """
+    - name: Get refspec to clone repo from rdo gerrit
+      ansible.builtin.set_fact:
+        change_refspec: >-
+            {{
+              repo | reproducer_refspec
+            }}
+"""
+
+RETURN = """
+  _value:
+    description: The refspec to pull the change.
+    type: str
+    sample:
+      "refs/changes/15/449415/5"
+
+"""
+
+from ansible.errors import AnsibleFilterError, AnsibleFilterTypeError
+
+
+class FilterModule:
+    @classmethod
+    def __map_repo_refspec(cls, repo):
+        if not isinstance(repo, dict):
+            raise AnsibleFilterTypeError(
+                f"reproducer_refspec requires a dict, got {type(repo)}"
+            )
+        if "change" not in repo:
+            # handle case pointing to main/master, e.g. a periodic job
+            return ""
+        change = repo["change"]
+        if "project" not in repo:
+            raise AnsibleFilterError(
+                "repo information does not contain 'project' field"
+            )
+        if "canonical_hostname" not in repo["project"]:
+            raise AnsibleFilterError(
+                "repo information does not contain 'canonical_hostname' field"
+            )
+        hostname = repo["project"]["canonical_hostname"]
+        if "rdoproject" in hostname or "code.eng" in hostname:
+            if "patchset" not in repo:
+                raise AnsibleFilterError(
+                    "repo information does not contain 'patchset' field"
+                )
+            patchset = repo["patchset"]
+            # changes coming from gerrit
+            return f"refs/changes/{change[-2:]}/{change}/{patchset}"
+        else:
+            # changes coming from github
+            return f"pull/{change}/head"
+
+    def filters(self):
+        return {
+            "reproducer_refspec": self.__map_repo_refspec,
+        }

roles/reproducer/tasks/push_code.yml

@@ -26,10 +26,12 @@
       loop: "{{ zuul['items'] | zip(repos_dir_stats.results) | list }}"
 
     - name: Fetch zuul.items repositories
+      vars:
+        _skip_refspec: "{{ job_id is not defined or repo.change is not defined }}"
       ansible.builtin.git:
         dest: "{{ repo.project.src_dir }}"
-        repo: "https://{{ repo.project.canonical_name }}"
-        refspec: "{{ omit if job_id is not defined else 'pull/'~repo.change~'/head:'~job_id }}"
+        repo: "https://{{ repo.project.canonical_hostname }}/{{ repo.project.canonical_hostname | reproducer_gerrit_infix }}{{ repo.project.name }}"
+        refspec: "{{ omit if _skip_refspec else repo | reproducer_refspec ~ ':' ~ job_id }}"
         version: "{{ job_id | default(omit) }}"
         force: true
       loop: "{{ zuul['items'] }}"

tests/integration/targets/filter_reproducer_gerrit_infix/tasks/main.yml

@@ -0,0 +1,23 @@
+- name: Test reproducer_gerrit_infix filter
+  ansible.builtin.assert:
+    that:
+      - "'github.com' | cifmw.general.reproducer_gerrit_infix == ''"
+      - "'review.rdoproject.org' | cifmw.general.reproducer_gerrit_infix == 'r/'"
+      - "'review.rdoprojetc.org' | cifmw.general.reproducer_gerrit_infix == ''"
+      - "'code.engineering.com' | cifmw.general.reproducer_gerrit_infix == 'gerrit/'"
+      - "'gitlab.cee' | cifmw.general.reproducer_gerrit_infix == ''"
+
+- name: Test reproducer_gerrit_infix bad argument
+  vars:
+    input:
+      - "github.com"
+  ansible.builtin.debug:
+    var: "input | cifmw.general.reproducer_gerrit_infix"
+  register: _bad_reproducer_gerrit_infix_argument
+  ignore_errors: true
+
+- name: Verify reproducer_gerrit_infix showed an error message
+  ansible.builtin.assert:
+    that:
+      - _bad_reproducer_gerrit_infix_argument is failed
+      - "'reproducer_gerrit_infix requires a str' in _bad_reproducer_gerrit_infix_argument.msg"

tests/integration/targets/filter_reproducer_refspec/tasks/main.yml

@@ -0,0 +1,133 @@
+- name: Test reproducer_refspec bad argument
+  vars:
+    input:
+      - "github.com"
+  ansible.builtin.debug:
+    var: "input | cifmw.general.reproducer_refspec"
+  register: _bad_reproducer_refspec_argument
+  ignore_errors: true
+
+- name: Verify reproducer_refspec showed an error message
+  ansible.builtin.assert:
+    that:
+      - _bad_reproducer_refspec_argument is failed
+      - "'reproducer_refspec requires a dict,' in _bad_reproducer_refspec_argument.msg"
+
+- name: Test reproducer_refspec bad argument, no project
+  vars:
+    input:
+      hostname: "github.com"
+      change: "abc"
+  ansible.builtin.debug:
+    var: "input | cifmw.general.reproducer_refspec"
+  register: _bad_reproducer_refspec_argument
+  ignore_errors: true
+
+- name: Verify reproducer_refspec showed an error message
+  ansible.builtin.assert:
+    that:
+      - _bad_reproducer_refspec_argument is failed
+      - "'repo information does not contain \\'project\\' field' in _bad_reproducer_refspec_argument.msg"
+
+- name: Test reproducer_refspec bad argument, no hostname
+  vars:
+    input:
+      project: {}
+      change: "abc"
+  ansible.builtin.debug:
+    var: "input | cifmw.general.reproducer_refspec"
+  register: _bad_reproducer_refspec_argument
+  ignore_errors: true
+
+- name: Verify reproducer_refspec showed an error message
+  ansible.builtin.assert:
+    that:
+      - _bad_reproducer_refspec_argument is failed
+      - "'repo information does not contain \\'canonical_hostname\\' field' in _bad_reproducer_refspec_argument.msg"
+
+- name: Test reproducer_refspec bad argument, no patchset
+  vars:
+    input:
+      project:
+        canonical_hostname: "rdoproject.com"
+      change: "abc"
+  ansible.builtin.debug:
+    var: "input | cifmw.general.reproducer_refspec"
+  register: _bad_reproducer_refspec_argument
+  ignore_errors: true
+
+- name: Verify reproducer_refspec showed an error message
+  ansible.builtin.assert:
+    that:
+      - _bad_reproducer_refspec_argument is failed
+      - "'repo information does not contain \\'patchset\\' field' in _bad_reproducer_refspec_argument.msg"
+
+- name: Test reproducer_refspec no patchset in github
+  vars:
+    input:
+      project:
+        canonical_hostname: "github.com"
+      change: "abc"
+  ansible.builtin.assert:
+    that:
+      - "input | cifmw.general.reproducer_refspec == 'pull/abc/head'"
+
+- name: Test reproducer_refspec no change
+  vars:
+    input:
+      project:
+        canonical_hostname: "github.com"
+  ansible.builtin.assert:
+    that:
+      - "input | cifmw.general.reproducer_refspec == ''"
+
+- name: Test reproducer_refspec periodic job
+  vars:
+    input:
+      project:
+        canonical_hostname: "review.rdoproject.org"
+  ansible.builtin.assert:
+    that:
+      - "input | cifmw.general.reproducer_refspec == ''"
+
+- name: Test reproducer_refspec github refspec
+  vars:
+    input:
+      project:
+        canonical_hostname: "github.org"
+      change: "1035"
+  ansible.builtin.assert:
+    that:
+      - "input | cifmw.general.reproducer_refspec == 'pull/1035/head'"
+
+- name: Test reproducer_refspec gitlab refspec
+  vars:
+    input:
+      project:
+        canonical_hostname: "gitlab.org"
+      change: "1035"
+  ansible.builtin.assert:
+    that:
+      - "input | cifmw.general.reproducer_refspec == 'pull/1035/head'"
+
+- name: Test reproducer_refspec gerrit refspec
+  vars:
+    input:
+      project:
+        canonical_hostname: "review.rdoproject.org"
+      change: "51319"
+      patchset: "6"
+  ansible.builtin.assert:
+    that:
+      - "input | cifmw.general.reproducer_refspec == 'refs/changes/19/51319/6'"
+
+- name: Test reproducer_refspec code.eng gerrit refspec
+  vars:
+    input:
+      project:
+        canonical_hostname: "review.code.eng.org"
+      change: "51319"
+      patchset: "6"
+  ansible.builtin.assert:
+    that:
+      - "input | cifmw.general.reproducer_refspec == 'refs/changes/19/51319/6'"