diff --git a/packit_service/constants.py b/packit_service/constants.py index ba4d0c5a4..36413cc67 100644 --- a/packit_service/constants.py +++ b/packit_service/constants.py @@ -36,6 +36,11 @@ ) TESTING_FARM_ARTIFACTS_KEY = "artifacts" +ELN_PACKAGE_LIST = "https://tiny.distro.builders/view-all-source-package-name-list--view-eln.txt" +ELN_EXTRAS_PACKAGE_LIST = ( + "https://tiny.distro.builders/view-all-source-package-name-list--view-eln-extras.txt" +) + MSG_DOWNSTREAM_JOB_ERROR_HEADER = ( "Packit failed on creating {object} in dist-git " "({dist_git_url}):\n\n" diff --git a/packit_service/utils.py b/packit_service/utils.py index 90eb6b767..342e31d1c 100644 --- a/packit_service/utils.py +++ b/packit_service/utils.py @@ -3,7 +3,8 @@ import logging import os -from datetime import datetime, timezone +import tempfile +from datetime import datetime, timedelta, timezone from io import StringIO from logging import StreamHandler from pathlib import Path @@ -11,12 +12,14 @@ from typing import Optional import requests +from cachetools.func import ttl_cache from ogr.abstract import PullRequest from packit.config import JobConfig, PackageConfig from packit.schema import JobConfigSchema, PackageConfigSchema from packit.utils import PackitFormatter from packit_service import __version__ as ps_version +from packit_service.constants import ELN_EXTRAS_PACKAGE_LIST, ELN_PACKAGE_LIST logger = logging.getLogger(__name__) @@ -298,3 +301,13 @@ def download_file(url: str, path: Path): return False return True + + +@ttl_cache(maxsize=1, ttl=timedelta(hours=12).seconds) +def get_eln_packages(): + packages = [] + for url in (ELN_PACKAGE_LIST, ELN_EXTRAS_PACKAGE_LIST): + with tempfile.NamedTemporaryFile() as tmp: + if download_file(url, tmp.name): + packages.extend(Path(tmp.name).read_text().splitlines()) + return packages diff --git a/packit_service/worker/checker/distgit.py b/packit_service/worker/checker/distgit.py index dc877ab8d..8e292b28f 100644 --- a/packit_service/worker/checker/distgit.py +++ b/packit_service/worker/checker/distgit.py @@ -5,7 +5,9 @@ import re from packit.config.aliases import get_branches +from packit.utils import commands +from packit_service import utils from packit_service.constants import MSG_GET_IN_TOUCH from packit_service.events import ( anitya, @@ -155,6 +157,20 @@ def pre_check(self) -> bool: return True +class PackageNeedsELNBuildFromRawhide(Checker, GetPagurePullRequestMixin): + def pre_check(self) -> bool: + if ( + self.pull_request.target_branch == "rawhide" + and self.project.repo in utils.get_eln_packages() + ): + repo_url = self.project.get_git_urls().get("git") + return not commands.run_command( + ["git", "ls-remote", repo_url, "eln"], output=True + ).stdout.strip() + + return False + + class HasIssueCommenterRetriggeringPermissions(ActorChecker): """To be able to retrigger a koji-build the issue commenter should have write permission on the project. diff --git a/packit_service/worker/handlers/abstract.py b/packit_service/worker/handlers/abstract.py index 992c1df9c..4dcee1718 100644 --- a/packit_service/worker/handlers/abstract.py +++ b/packit_service/worker/handlers/abstract.py @@ -250,6 +250,7 @@ class TaskName(str, enum.Enum): openscanhub_task_started = "task.openscanhub_task_started" downstream_koji_scratch_build = "task.run_downstream_koji_scratch_build_handler" downstream_koji_scratch_build_report = "task.run_downstream_koji_scratch_build_report_handler" + downstream_koji_eln_scratch_build = "task.run_downstream_koji_eln_scratch_build_handler" downstream_log_detective_results = "task.run_downstream_log_detective_results_handler" diff --git a/packit_service/worker/handlers/distgit.py b/packit_service/worker/handlers/distgit.py index 13ccd6536..dd3806656 100644 --- a/packit_service/worker/handlers/distgit.py +++ b/packit_service/worker/handlers/distgit.py @@ -80,6 +80,7 @@ IsProjectOk, IsUpstreamTagMatchingConfig, LabelsOnDistgitPR, + PackageNeedsELNBuildFromRawhide, PermissionOnDistgit, PermissionOnDistgitForFedoraCI, TaggedBuildIsNotABuildOfSelf, @@ -792,7 +793,6 @@ def __init__( job_config: JobConfig, event: dict, celery_task: Task, - koji_group_model_id: Optional[int] = None, ): super().__init__( package_config=package_config, @@ -802,7 +802,6 @@ def __init__( ) self._project_url = self.data.project_url self._packit_api = None - self._koji_group_model_id = koji_group_model_id self._ci_helper: Optional[FedoraCIHelper] = None @property @@ -966,6 +965,21 @@ def run_koji_build( ).stdout +@run_for_comment_as_fedora_ci(command="scratch-build") +@reacts_to_as_fedora_ci(event=pagure.pr.Action) +@reacts_to_as_fedora_ci(event=pagure.pr.Comment) +class DownstreamKojiELNScratchBuildHandler(DownstreamKojiScratchBuildHandler): + task_name = TaskName.downstream_koji_eln_scratch_build + + @property + def dist_git_branch(self) -> str: + return "eln" + + @staticmethod + def get_checkers() -> tuple[type[Checker], ...]: + return (PermissionOnDistgitForFedoraCI, PackageNeedsELNBuildFromRawhide) + + class AbstractDownstreamKojiBuildHandler( abc.ABC, RetriableJobHandler, diff --git a/packit_service/worker/tasks.py b/packit_service/worker/tasks.py index 522b2689d..2373d62a5 100644 --- a/packit_service/worker/tasks.py +++ b/packit_service/worker/tasks.py @@ -74,6 +74,7 @@ ) from packit_service.worker.handlers.distgit import ( DownstreamKojiBuildHandler, + DownstreamKojiELNScratchBuildHandler, DownstreamKojiScratchBuildHandler, PullFromUpstreamHandler, RetriggerDownstreamKojiBuildHandler, @@ -464,6 +465,24 @@ def run_downstream_koji_scratch_build_handler( return get_handlers_task_results(handler.run_job(), event) +@celery_app.task( + bind=True, + name=TaskName.downstream_koji_eln_scratch_build, + base=TaskWithRetry, + queue="long-running", +) +def run_downstream_koji_eln_scratch_build_handler( + self, event: dict, package_config: dict, job_config: dict +): + handler = DownstreamKojiELNScratchBuildHandler( + package_config=load_package_config(package_config), + job_config=load_job_config(job_config), + event=event, + celery_task=self, + ) + return get_handlers_task_results(handler.run_job(), event) + + @celery_app.task( name=TaskName.sync_from_downstream, base=TaskWithRetry, diff --git a/tests/integration/test_dg_pr.py b/tests/integration/test_dg_pr.py index afc7c6219..5322c1d8b 100644 --- a/tests/integration/test_dg_pr.py +++ b/tests/integration/test_dg_pr.py @@ -14,6 +14,7 @@ from packit.local_project import LocalProjectBuilder from packit.utils import commands +from packit_service import utils from packit_service.config import ServiceConfig from packit_service.constants import SANDCASTLE_WORK_DIR from packit_service.models import ( @@ -28,6 +29,7 @@ from packit_service.worker.jobs import SteveJobs from packit_service.worker.monitoring import Pushgateway from packit_service.worker.tasks import ( + run_downstream_koji_eln_scratch_build_handler, run_downstream_koji_scratch_build_handler, ) from tests.spellbook import DATA_DIR, first_dict_value, get_parameters_from_results @@ -39,30 +41,47 @@ def distgit_pr_event(): @pytest.mark.parametrize( - "target_branch, uid, check_name", + "target_branch, uid, check_name, eln", [ pytest.param( "rawhide", "e0091d5fbcb20572cbf2e6442af9bed5", "Packit - scratch build - rawhide", + False, id="rawhide target branch", ), + pytest.param( + "rawhide", + "8edd48272efe6aff7d1d92bdffcaf9a0", + "Packit - scratch build - eln", + True, + id="rawhide branch, rawhide + eln target", + ), pytest.param( "f42", "6f08c3bbb20660dc8c597bc7dbe4f056", "Packit - scratch build - f42", + False, id="f42 target branch", ), ], ) -def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, check_name): +def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, check_name, eln): distgit_pr_event["pullrequest"]["branch"] = target_branch pr_object = ( - flexmock() + flexmock(target_branch=target_branch) .should_receive("set_flag") .with_args(username=check_name, comment=str, url=str, status=CommitStatus, uid=uid) .mock() ) + if eln: + check_name = "Packit - scratch build - rawhide" + uid = "e0091d5fbcb20572cbf2e6442af9bed5" + ( + pr_object.should_receive("set_flag") + .with_args(username=check_name, comment=str, url=str, status=CommitStatus, uid=uid) + .mock() + ) dg_project = ( flexmock( PagureProject(namespace="rpms", repo="optee_os", service=flexmock(read_only=False)) @@ -73,6 +92,9 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che .should_receive("get_pr") .and_return(pr_object) .mock() + .should_receive("get_git_urls") + .and_return({"git": "https://src.fedoraproject.org/rpms/optee_os.git"}) + .mock() ) service_config = ( flexmock( @@ -112,6 +134,13 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che ) flexmock(PipelineModel).should_receive("create") + flexmock(utils).should_receive("get_eln_packages").and_return(["optee_os"] if eln else []) + if eln: + flexmock(commands).should_receive("run_command").with_args( + ["git", "ls-remote", "https://src.fedoraproject.org/rpms/optee_os.git", "eln"], + output=True, + ).and_return(flexmock(stdout="")) + koji_build = flexmock( id=123, target="main", @@ -130,8 +159,8 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che ) flexmock(LocalProjectBuilder, _refresh_the_state=lambda *args: None) - flexmock(Signature).should_receive("apply_async").once() - flexmock(Pushgateway).should_receive("push").times(2).and_return() + flexmock(Signature).should_receive("apply_async").times(2 if eln else 1) + flexmock(Pushgateway).should_receive("push").times(3 if eln else 2).and_return() flexmock(commands).should_receive("run_command_remote").with_args( cmd=[ "koji", @@ -145,15 +174,29 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che output=True, print_live=True, ).and_return(flexmock(stdout="some output")) + if eln: + flexmock(commands).should_receive("run_command_remote").with_args( + cmd=[ + "koji", + "build", + "--scratch", + "--nowait", + "eln", + "git+https://src.fedoraproject.org/forks/zbyszek/rpms/optee_os.git#889f07af35d27bbcaf9c535c17a63b974aa42ee3", + ], + cwd=Path, + output=True, + print_live=True, + ).and_return(flexmock(stdout="some output")) flexmock(PackitAPI).should_receive("init_kerberos_ticket") flexmock(distgit).should_receive("get_koji_task_id_and_url_from_stdout").and_return( (123, "koji-web-url") - ).once() + ).times(2 if eln else 1) processing_results = SteveJobs().process_message(distgit_pr_event) event_dict, _, job_config, package_config = get_parameters_from_results( - processing_results, + processing_results[:1], ) assert json.dumps(event_dict) results = run_downstream_koji_scratch_build_handler( @@ -163,3 +206,12 @@ def test_downstream_koji_scratch_build(distgit_pr_event, target_branch, uid, che ) assert first_dict_value(results["job"])["success"] + + if eln: + results = run_downstream_koji_eln_scratch_build_handler( + package_config=package_config, + event=event_dict, + job_config=job_config, + ) + + assert first_dict_value(results["job"])["success"]