upstream_issue.py

# This file is part of sync2jira.
# Copyright (C) 2016 Red Hat, Inc.
#
# sync2jira is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# sync2jira is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with sync2jira; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110.15.0 USA
#
# Authors:  Ralph Bean <rbean@redhat.com>

from copy import deepcopy
import logging
from urllib.parse import urlencode

from github import Github, UnknownObjectException
import requests

import sync2jira.intermediary as i

log = logging.getLogger("sync2jira")
graphqlurl = "https://api.github.com/graphql"

ghquery = """
    query MyQuery(
        $orgname: String!, $reponame: String!, $issuenumber: Int!
    ) {
        repository(owner: $orgname, name: $reponame) {
          issue(number: $issuenumber) {
            title
            body
            projectItems(first: 3) {
              nodes {
                project {
                  title
                  number
                  url
                }
                fieldValues(first: 100) {
                  nodes {
                    ... on ProjectV2ItemFieldSingleSelectValue {
                      name
                      fieldName: field {
                        ... on ProjectV2FieldCommon {
                          name
                        }
                      }
                    }
                    ... on ProjectV2ItemFieldTextValue {
                      text
                      fieldName: field {
                        ... on ProjectV2FieldCommon {
                          name
                        }
                      }
                    }
                    ... on ProjectV2ItemFieldNumberValue {
                      number
                      fieldName: field {
                        ... on ProjectV2FieldCommon {
                          name
                        }
                      }
                    }
                    ... on ProjectV2ItemFieldDateValue {
                      date
                      fieldName: field {
                        ... on ProjectV2FieldCommon {
                          name
                        }
                      }
                    }
                    ... on ProjectV2ItemFieldUserValue {
                      users(first: 10) {
                        nodes {
                          login
                        }
                      }
                      fieldName: field {
                        ... on ProjectV2FieldCommon {
                          name
                        }
                      }
                    }
                    ... on ProjectV2ItemFieldIterationValue {
                      title
                      duration
                      startDate
                      fieldName: field {
                        ... on ProjectV2FieldCommon {
                          name
                        }
                      }
                    }
                  }
                }
              }
            }
          }
        }
      }
"""

ghquery_pr = ghquery.replace(
    "issue(number: $issuenumber)", "pullRequest(number: $issuenumber)"
)


def get_github_client(config):
    """
    Helper function returning headers and github_client built from config.

    :param dict config: Config
    :returns: (headers, github_client)
    :rtype: tuple
    """

    token = config["sync2jira"].get("github_token")
    headers = {"Authorization": "token " + token} if token else {}
    github_client = Github(token, retry=5)
    return headers, github_client


def passes_github_filters(item, config, upstream, item_type="issue"):
    """
    Apply GitHub filters (labels, milestone, other fields) to an item.

    :param dict item: GitHub issue or PR data
    :param dict _filter: Filter configuration
    :param str upstream: Upstream repository name
    :param str item_type: Type of item for logging ("issue" or "PR")
    :returns: True if item passes all filters, False otherwise
    :rtype: bool
    """
    filter_config = (
        config["sync2jira"].get("filters", {}).get("github", {}).get(upstream, {})
    )
    mapped_repos = config["sync2jira"]["map"]["github"]
    if upstream not in mapped_repos:
        log.debug("%r not in Github map: %r", upstream, mapped_repos.keys())
        return None
    key = "pullrequest" if item_type == "PR" else "issue"
    if key not in mapped_repos[upstream].get("sync", []):
        log.debug(
            "%r not in Github sync map: %r",
            key,
            mapped_repos[upstream].get("sync", []),
        )
        return None

    for key, expected in filter_config.items():
        if key == "labels":
            # special handling for label: we look for it in the list of labels
            actual = {label["name"] for label in item.get("labels", [])}
            if actual.isdisjoint(expected):
                log.debug(
                    "Labels %s not found on %s: %s", expected, upstream, item_type
                )
                return False
        elif key == "milestone":
            # special handling for milestone: use the number
            milestone = item.get(key) or {}  # Key might exist with value `None`
            actual = milestone.get("number")
            if expected != actual:
                log.debug(
                    "Milestone %s not set on %s: %s", expected, upstream, item_type
                )
                return False
        else:
            # direct comparison
            actual = item.get(key)
            if actual != expected:
                log.debug(
                    "Actual %r %r != expected %r on %s %s",
                    key,
                    actual,
                    expected,
                    upstream,
                    item_type,
                )
                return False
    return True


def handle_github_message(body, config, is_pr=False):
    """
    Handle GitHub message from FedMsg.

    :param Dict body: FedMsg Message body
    :param Dict config: Config File
    :param Bool is_pr: msg refers to a pull request
    :returns: Issue object
    :rtype: sync2jira.intermediary.Issue
    """
    owner = body["repository"]["owner"]["login"]
    repo = body["repository"]["name"]
    upstream = "{owner}/{repo}".format(owner=owner, repo=repo)

    issue = body["issue"]
    if not passes_github_filters(issue, config, upstream, item_type="issue"):
        return None
    if is_pr and not issue.get("closed_at"):
        log.debug(
            "%r is a pull request.  Ignoring.", issue.get("html_url", "<missing URL>")
        )
        return None

    headers, github_client = get_github_client(config)
    reformat_github_issue(issue, upstream, github_client)
    updates_key = "pr_updates" if is_pr else "issue_updates"
    add_project_values(issue, upstream, headers, config, updates_key)
    return i.Issue.from_github(upstream, issue, config)


def github_issues(upstream, config):
    """
    Returns a generator for all GitHub issues in upstream repo.

    :param String upstream: Upstream Repo
    :param Dict config: Config Dict
    :returns: a generator for GitHub Issue objects
    :rtype: Generator[sync2jira.intermediary.Issue]
    """
    headers, github_client = get_github_client(config)
    for issue in generate_github_items("issues", upstream, config):
        if "pull_request" in issue or "/pull/" in issue.get("html_url", ""):
            # We don't want to copy these around
            orgname, reponame = upstream.rsplit("/", 1)
            log.debug(
                "Issue %s/%s#%s is a pull request; skipping",
                orgname,
                reponame,
                issue["number"],
            )
            continue
        reformat_github_issue(issue, upstream, github_client)
        add_project_values(issue, upstream, headers, config)
        yield i.Issue.from_github(upstream, issue, config)


def add_project_values(issue, upstream, headers, config, updates_key="issue_updates"):
    """Add values to an issue/PR from its corresponding card in a GitHub Project

    :param dict issue: Issue or PR dict
    :param str upstream: Upstream repo name
    :param dict headers: HTTP Request headers, including access token, if any
    :param dict config: Config
    :param str updates_key: Config key for the updates list
    """
    upstream_config = config["sync2jira"]["map"]["github"][upstream]
    updates = upstream_config.get(updates_key, [])
    github_project_fields = upstream_config.get("github_project_fields")
    if not github_project_fields or "github_project_fields" not in updates:
        log.debug(
            "github_project_fields is None or empty, skipping project field updates"
        )
        return
    project_number = upstream_config.get("github_project_number")
    issue["storypoints"] = None
    issue["priority"] = None
    issuenumber = issue["number"]
    orgname, reponame = upstream.rsplit("/", 1)
    variables = {"orgname": orgname, "reponame": reponame, "issuenumber": issuenumber}
    query = ghquery_pr if updates_key == "pr_updates" else ghquery
    response = requests.post(
        graphqlurl, headers=headers, json={"query": query, "variables": variables}
    )
    if response.status_code != 200:
        log.info(
            "HTTP error while fetching %s %s/%s#%s: %s",
            "PR" if updates_key == "pr_updates" else "issue",
            orgname,
            reponame,
            issuenumber,
            response.text,
        )
        return
    data = response.json()
    repo_data = data.get("data", {}).get("repository", {})
    gh_item = repo_data.get("pullRequest" if updates_key == "pr_updates" else "issue")
    if not gh_item:
        log.info(
            "GitHub error while fetching %s %s/%s#%s: %s",
            "PR" if updates_key == "pr_updates" else "issue",
            orgname,
            reponame,
            issuenumber,
            response.text,
        )
        return
    project_node = _get_current_project_node(
        upstream, project_number, issuenumber, gh_item
    )
    if not project_node:
        return
    item_nodes = project_node.get("fieldValues", {}).get("nodes", {})
    for item in item_nodes:
        gh_field_name = item.get("fieldName", {}).get("name")
        if not gh_field_name:
            continue
        prio_field = github_project_fields.get("priority", {}).get("gh_field")
        if gh_field_name == prio_field:
            issue["priority"] = item.get("name")
            continue
        sp_dict = github_project_fields.get("storypoints", {})
        sp_field = sp_dict.get("gh_field")
        if gh_field_name == sp_field:
            # Check if there's an options mapping (for Single Select fields); if
            # so, convert...
            if sp_options := sp_dict.get("options"):
                # Single Select field - get name and map it
                sp_value = item.get("name")
                if not sp_value:
                    log.warning(
                        "No Single Select name found for storypoints options in message for issue %s/%s#%s",
                        orgname,
                        reponame,
                        issuenumber,
                    )
                elif (sp_number := sp_options.get(sp_value)) is None:
                    log.info(
                        "Storypoints value '%s' not found in options mapping for issue %s/%s#%s",
                        sp_value,
                        orgname,
                        reponame,
                        issuenumber,
                    )
                else:
                    try:
                        issue["storypoints"] = int(sp_number)
                    except (ValueError, TypeError) as err:
                        log.info(
                            "Error converting Single Select storypoints value '%s' to int for issue %s/%s#%s: %s",
                            sp_number,
                            orgname,
                            reponame,
                            issuenumber,
                            err,
                        )
            else:
                # Number field - get number directly
                try:
                    issue["storypoints"] = int(item["number"])
                except (ValueError, TypeError, KeyError) as err:
                    log.info(
                        "Error converting Number field storypoints value '%s' to int for issue %s/%s#%s: %s",
                        item.get("number", "missing"),
                        orgname,
                        reponame,
                        issuenumber,
                        err,
                    )
            continue


def reformat_github_issue(issue, upstream, github_client):
    """Tweak Issue data format to better match Pagure"""

    # Update comments:
    # If there are no comments just make an empty array
    if not issue["comments"]:
        issue["comments"] = []
    else:
        # We have multiple comments and need to make api call to get them
        try:
            repo = github_client.get_repo(upstream)
        except UnknownObjectException:
            logging.warning(
                "GitHub repo %r not found (has it been deleted or made private?)",
                upstream,
            )
            raise
        github_issue = repo.get_issue(number=issue["number"])
        issue["comments"] = reformat_github_comments(github_issue.get_comments())

    # Update the rest of the parts
    reformat_github_common(issue, github_client)


def reformat_github_comments(comments):
    """Helper function which encapsulates reformatting comments"""
    return [
        {
            "author": comment.user.name or comment.user.login,
            "name": comment.user.login,
            "body": comment.body,
            "id": comment.id,
            "date_created": comment.created_at,
            "changed": None,
        }
        for comment in comments
    ]


def reformat_github_common(item, github_client):
    """Helper function which tweaks the data format of the parts of Issues and
    PRs which are common so that they better match Pagure
    """
    # Update reporter:
    # Search for the user
    reporter = github_client.get_user(item["user"]["login"])
    # Update the reporter field in the message (to match Pagure format)
    if reporter.name and reporter.name != "None":
        item["user"]["fullname"] = reporter.name
    else:
        item["user"]["fullname"] = item["user"]["login"]

    # Update assignee(s):
    assignees = []
    for person in item.get("assignees", []):
        assignee = github_client.get_user(person["login"])
        entry = {"login": person["login"]}
        if assignee.name and assignee.name != "None":
            entry["fullname"] = assignee.name
        assignees.append(entry)
    # Update the assignee field in the message (to match Pagure format)
    item["assignees"] = assignees

    # Update the label field in the message (to match Pagure format)
    if item["labels"]:
        # Loop through all the labels on GitHub and add them
        # to the new label list and then reassign the message
        new_label = []
        for label in item["labels"]:
            new_label.append(label["name"])
        item["labels"] = new_label

    # Update the milestone field in the message (to match Pagure format)
    if item.get("milestone"):
        item["milestone"] = item["milestone"]["title"]


def generate_github_items(api_method, upstream, config):
    """
    Returns a generator which yields all GitHub issues in upstream repo.

    :param String api_method: API method name
    :param String upstream: Upstream Repo
    :param Dict config: Config Dict
    :returns: a generator for GitHub Issue/PR objects
    :rtype: Generator[Any, Any, None]
    """
    token = config["sync2jira"].get("github_token")
    if not token:
        headers = {}
        log.warning("No github_token found.  We will be rate-limited...")
    else:
        headers = {"Authorization": "token " + token}

    params = config["sync2jira"].get("filters", {}).get("github", {}).get(upstream, {})

    url = "https://api.github.com/repos/" + upstream + "/" + api_method
    if params:
        labels = params.get("labels")
        if isinstance(labels, list):
            # We have to flatten the labels list to a comma-separated string,
            # so make a copy to avoid mutating the config object
            url_filter = deepcopy(params)
            url_filter["labels"] = ",".join(labels)
        else:
            url_filter = params  # Use the existing filter, unmodified
        url += "?" + urlencode(url_filter)

    return get_all_github_data(url, headers)


def _get_current_project_node(upstream, project_number, issue_number, gh_issue):
    project_items = gh_issue["projectItems"]["nodes"]
    # If there are no project items, there is nothing to return.
    if len(project_items) == 0:
        log.debug(
            "Issue %s#%s is not associated with any project", upstream, issue_number
        )
        return None

    if not project_number:
        # We don't have a configured project.  If there is exactly one project
        # item, we'll assume it's the right one and return it.
        if len(project_items) == 1:
            return project_items[0]

        # There are multiple projects associated with this issue; since we
        # don't have a configured project, we don't know which one to return,
        # so return none.
        prj = (f"{x['project']['url']}: {x['project']['title']}" for x in project_items)
        log.debug(
            "Project number is not configured, and the issue %s#%s"
            " is associated with more than one project: %s",
            upstream,
            issue_number,
            ", ".join(prj),
        )
        return None

    # Return the associated project which matches the configured project if we
    # can find it.
    for item in project_items:
        if item["project"]["number"] == project_number:
            return item

    log.debug(
        "Issue %s#%s is associated with multiple projects, "
        "but none match the configured project.",
        upstream,
        issue_number,
    )
    return None


def get_all_github_data(url, headers):
    """A generator which returns each response from a paginated GitHub API call"""
    link = {"next": url}
    while "next" in link:
        response = api_call_get(link["next"], headers=headers)
        for issue in response.json():
            comments = api_call_get(issue["comments_url"], headers=headers)
            issue["comments"] = comments.json()
            yield issue
        link = _github_link_field_to_dict(response.headers.get("link"))


def _github_link_field_to_dict(field):
    """Utility for ripping apart GitHub's Link header field."""

    if not field:
        return {}
    return dict(
        (part.split("; ")[1][5:-1], part.split("; ")[0][1:-1])
        for part in field.split(", ")
    )


def api_call_get(url, **kwargs):
    """Helper function to encapsulate a REST API GET call"""
    response = requests.get(url, **kwargs)
    if not bool(response):
        # noinspection PyBroadException
        try:
            reason = response.json()
        except Exception:
            reason = response.text
        raise IOError("response: %r %r %r" % (response, reason, response.request.url))
    return response