Trigger ELN scratch builds from rawhide

Signed-off-by: Nikola Forró <[email protected]>

Author: nforro

packit_service/constants.py

@@ -36,6 +36,11 @@
 )
 TESTING_FARM_ARTIFACTS_KEY = "artifacts"
 
+ELN_PACKAGE_LIST = "https://tiny.distro.builders/view-all-source-package-name-list--view-eln.txt"
+ELN_EXTRAS_PACKAGE_LIST = (
+    "https://tiny.distro.builders/view-all-source-package-name-list--view-eln-extras.txt"
+)
+
 MSG_DOWNSTREAM_JOB_ERROR_HEADER = (
     "Packit failed on creating {object} in dist-git "
     "({dist_git_url}):\n\n"

packit_service/utils.py

@@ -3,20 +3,23 @@
 
 import logging
 import os
-from datetime import datetime, timezone
+import tempfile
+from datetime import datetime, timedelta, timezone
 from io import StringIO
 from logging import StreamHandler
 from pathlib import Path
 from re import search
 from typing import Optional
 
 import requests
+from cachetools.func import ttl_cache
 from ogr.abstract import PullRequest
 from packit.config import JobConfig, PackageConfig
 from packit.schema import JobConfigSchema, PackageConfigSchema
 from packit.utils import PackitFormatter
 
 from packit_service import __version__ as ps_version
+from packit_service.constants import ELN_EXTRAS_PACKAGE_LIST, ELN_PACKAGE_LIST
 
 logger = logging.getLogger(__name__)
 
@@ -298,3 +301,13 @@ def download_file(url: str, path: Path):
         return False
 
     return True
+
+
+@ttl_cache(maxsize=1, ttl=timedelta(hours=12).seconds)
+def get_eln_packages():
+    packages = []
+    for url in (ELN_PACKAGE_LIST, ELN_EXTRAS_PACKAGE_LIST):
+        with tempfile.NamedTemporaryFile() as tmp:
+            if download_file(url, tmp.name):
+                packages.extend(Path(tmp.name).read_text().splitlines())
+    return packages

packit_service/worker/checker/distgit.py

@@ -5,7 +5,9 @@
 import re
 
 from packit.config.aliases import get_branches
+from packit.utils import commands
 
+from packit_service import utils
 from packit_service.constants import MSG_GET_IN_TOUCH
 from packit_service.events import (
     anitya,
@@ -155,6 +157,20 @@ def pre_check(self) -> bool:
         return True
 
 
+class PackageNeedsELNBuildFromRawhide(Checker, GetPagurePullRequestMixin):
+    def pre_check(self) -> bool:
+        if (
+            self.pull_request.target_branch == "rawhide"
+            and self.project.repo in utils.get_eln_packages()
+        ):
+            repo_url = self.project.get_git_urls().get("git")
+            return not commands.run_command(
+                ["git", "ls-remote", repo_url, "eln"], output=True
+            ).stdout.strip()
+
+        return False
+
+
 class HasIssueCommenterRetriggeringPermissions(ActorChecker):
     """To be able to retrigger a koji-build the issue commenter should
     have write permission on the project.

packit_service/worker/handlers/abstract.py

@@ -250,6 +250,7 @@ class TaskName(str, enum.Enum):
     openscanhub_task_started = "task.openscanhub_task_started"
     downstream_koji_scratch_build = "task.run_downstream_koji_scratch_build_handler"
     downstream_koji_scratch_build_report = "task.run_downstream_koji_scratch_build_report_handler"
+    downstream_koji_eln_scratch_build = "task.run_downstream_koji_eln_scratch_build_handler"
     downstream_log_detective_results = "task.run_downstream_log_detective_results_handler"
 
 

packit_service/worker/handlers/distgit.py

@@ -80,6 +80,7 @@
     IsProjectOk,
     IsUpstreamTagMatchingConfig,
     LabelsOnDistgitPR,
+    PackageNeedsELNBuildFromRawhide,
     PermissionOnDistgit,
     PermissionOnDistgitForFedoraCI,
     TaggedBuildIsNotABuildOfSelf,
@@ -792,7 +793,6 @@ def __init__(
         job_config: JobConfig,
         event: dict,
         celery_task: Task,
-        koji_group_model_id: Optional[int] = None,
     ):
         super().__init__(
             package_config=package_config,
@@ -802,7 +802,6 @@ def __init__(
         )
         self._project_url = self.data.project_url
         self._packit_api = None
-        self._koji_group_model_id = koji_group_model_id
         self._ci_helper: Optional[FedoraCIHelper] = None
 
     @property
@@ -966,6 +965,21 @@ def run_koji_build(
         ).stdout
 
 
+@run_for_comment_as_fedora_ci(command="scratch-build")
+@reacts_to_as_fedora_ci(event=pagure.pr.Action)
+@reacts_to_as_fedora_ci(event=pagure.pr.Comment)
+class DownstreamKojiELNScratchBuildHandler(DownstreamKojiScratchBuildHandler):
+    task_name = TaskName.downstream_koji_eln_scratch_build
+
+    @property
+    def dist_git_branch(self) -> str:
+        return "eln"
+
+    @staticmethod
+    def get_checkers() -> tuple[type[Checker], ...]:
+        return (PermissionOnDistgitForFedoraCI, PackageNeedsELNBuildFromRawhide)
+
+
 class AbstractDownstreamKojiBuildHandler(
     abc.ABC,
     RetriableJobHandler,

packit_service/worker/tasks.py

@@ -74,6 +74,7 @@
 )
 from packit_service.worker.handlers.distgit import (
     DownstreamKojiBuildHandler,
+    DownstreamKojiELNScratchBuildHandler,
     DownstreamKojiScratchBuildHandler,
     PullFromUpstreamHandler,
     RetriggerDownstreamKojiBuildHandler,
@@ -464,6 +465,24 @@ def run_downstream_koji_scratch_build_handler(
     return get_handlers_task_results(handler.run_job(), event)
 
 
+@celery_app.task(
+    bind=True,
+    name=TaskName.downstream_koji_eln_scratch_build,
+    base=TaskWithRetry,
+    queue="long-running",
+)
+def run_downstream_koji_eln_scratch_build_handler(
+    self, event: dict, package_config: dict, job_config: dict
+):
+    handler = DownstreamKojiELNScratchBuildHandler(
+        package_config=load_package_config(package_config),
+        job_config=load_job_config(job_config),
+        event=event,
+        celery_task=self,
+    )
+    return get_handlers_task_results(handler.run_job(), event)
+
+
 @celery_app.task(
     name=TaskName.sync_from_downstream,
     base=TaskWithRetry,

tests/integration/test_dg_pr.py

@@ -14,6 +14,7 @@
 from packit.local_project import LocalProjectBuilder
 from packit.utils import commands
 
+from packit_service import utils
 from packit_service.config import ServiceConfig
 from packit_service.constants import SANDCASTLE_WORK_DIR
 from packit_service.models import (
@@ -28,6 +29,7 @@
 from packit_service.worker.jobs import SteveJobs
 from packit_service.worker.monitoring import Pushgateway
 from packit_service.worker.tasks import (
+    run_downstream_koji_eln_scratch_build_handler,
     run_downstream_koji_scratch_build_handler,
 )
 from tests.spellbook import DATA_DIR, first_dict_value, get_parameters_from_results
@@ -39,30 +41,47 @@ def distgit_pr_event():
 
 
 @pytest.mark.parametrize(
-    "target_branch, uid, check_name",
+    "target_branch, uid, check_name, eln",
     [
         pytest.param(
             "rawhide",
             "e0091d5fbcb20572cbf2e6442af9bed5",
             "Packit - scratch build - rawhide",
+            False,
             id="rawhide target branch",
         ),
+        pytest.param(
+            "rawhide",
+            "8edd48272efe6aff7d1d92bdffcaf9a0",
+            "Packit - scratch build - eln",
+            True,
+            id="rawhide branch, rawhide + eln target",
+        ),
         pytest.param(
             "f42",
             "6f08c3bbb20660dc8c597bc7dbe4f056",
             "Packit - scratch build - f42",
+            False,
             id="f42 target branch",
         ),
     ],
 )
-def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, check_name):
+def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, check_name, eln):
     distgit_pr_event["pullrequest"]["branch"] = target_branch
     pr_object = (
-        flexmock()
+        flexmock(target_branch=target_branch)
         .should_receive("set_flag")
         .with_args(username=check_name, comment=str, url=str, status=CommitStatus, uid=uid)
         .mock()
     )
+    if eln:
+        check_name = "Packit - scratch build - rawhide"
+        uid = "e0091d5fbcb20572cbf2e6442af9bed5"
+        (
+            pr_object.should_receive("set_flag")
+            .with_args(username=check_name, comment=str, url=str, status=CommitStatus, uid=uid)
+            .mock()
+        )
     dg_project = (
         flexmock(
             PagureProject(namespace="rpms", repo="optee_os", service=flexmock(read_only=False))
@@ -73,6 +92,9 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che
         .should_receive("get_pr")
         .and_return(pr_object)
         .mock()
+        .should_receive("get_git_urls")
+        .and_return({"git": "https://src.fedoraproject.org/rpms/optee_os.git"})
+        .mock()
     )
     service_config = (
         flexmock(
@@ -112,6 +134,13 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che
     )
     flexmock(PipelineModel).should_receive("create")
 
+    flexmock(utils).should_receive("get_eln_packages").and_return(["optee_os"] if eln else [])
+    if eln:
+        flexmock(commands).should_receive("run_command").with_args(
+            ["git", "ls-remote", "https://src.fedoraproject.org/rpms/optee_os.git", "eln"],
+            output=True,
+        ).and_return(flexmock(stdout=""))
+
     koji_build = flexmock(
         id=123,
         target="main",
@@ -130,8 +159,8 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che
     )
 
     flexmock(LocalProjectBuilder, _refresh_the_state=lambda *args: None)
-    flexmock(Signature).should_receive("apply_async").once()
-    flexmock(Pushgateway).should_receive("push").times(2).and_return()
+    flexmock(Signature).should_receive("apply_async").times(2 if eln else 1)
+    flexmock(Pushgateway).should_receive("push").times(3 if eln else 2).and_return()
     flexmock(commands).should_receive("run_command_remote").with_args(
         cmd=[
             "koji",
@@ -145,15 +174,29 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che
         output=True,
         print_live=True,
     ).and_return(flexmock(stdout="some output"))
+    if eln:
+        flexmock(commands).should_receive("run_command_remote").with_args(
+            cmd=[
+                "koji",
+                "build",
+                "--scratch",
+                "--nowait",
+                "eln",
+                "git+https://src.fedoraproject.org/forks/zbyszek/rpms/optee_os.git#889f07af35d27bbcaf9c535c17a63b974aa42ee3",
+            ],
+            cwd=Path,
+            output=True,
+            print_live=True,
+        ).and_return(flexmock(stdout="some output"))
     flexmock(PackitAPI).should_receive("init_kerberos_ticket")
 
     flexmock(distgit).should_receive("get_koji_task_id_and_url_from_stdout").and_return(
         (123, "koji-web-url")
-    ).once()
+    ).times(2 if eln else 1)
 
     processing_results = SteveJobs().process_message(distgit_pr_event)
     event_dict, _, job_config, package_config = get_parameters_from_results(
-        processing_results,
+        processing_results[:1],
     )
     assert json.dumps(event_dict)
     results = run_downstream_koji_scratch_build_handler(
@@ -163,3 +206,12 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che
     )
 
     assert first_dict_value(results["job"])["success"]
+
+    if eln:
+        results = run_downstream_koji_eln_scratch_build_handler(
+            package_config=package_config,
+            event=event_dict,
+            job_config=job_config,
+        )
+
+        assert first_dict_value(results["job"])["success"]