diff --git a/database/enums.py b/database/enums.py index dae9c7afc..35b214846 100644 --- a/database/enums.py +++ b/database/enums.py @@ -26,6 +26,16 @@ class Notification(Enum): codecov_slack_app = "codecov_slack_app" +notification_type_status_or_checks = { + Notification.status_changes, + Notification.status_patch, + Notification.status_project, + Notification.checks_changes, + Notification.checks_patch, + Notification.checks_project, +} + + class NotificationState(Enum): pending = "pending" success = "success" diff --git a/services/notification/__init__.py b/services/notification/__init__.py index ebe61543e..650ddfb18 100644 --- a/services/notification/__init__.py +++ b/services/notification/__init__.py @@ -15,6 +15,7 @@ from shared.torngit.base import TorngitBaseAdapter from shared.yaml import UserYaml +from database.enums import notification_type_status_or_checks from database.models.core import GITHUB_APP_INSTALLATION_DEFAULT_NAME, Owner, Repository from services.comparison import ComparisonProxy from services.decoration import Decoration @@ -36,6 +37,7 @@ ChecksWithFallback, ) from services.notification.notifiers.codecov_slack_app import CodecovSlackAppNotifier +from services.notification.notifiers.mixins.status import StatusState from services.yaml import read_yaml_field from services.yaml.reader import get_components_from_yaml @@ -254,15 +256,62 @@ def notify(self, comparison: ComparisonProxy) -> list[IndividualResult]: repoid=comparison.head.commit.repoid, ), ) - results = [ - self.notify_individual_notifier(notifier, comparison) + + results = [] + status_or_checks_helper_text = {} + notifiers_to_notify = [ + notifier for notifier in self.get_notifiers_instances() if notifier.is_enabled() ] + + status_or_checks_notifiers = [ + notifier + for notifier in notifiers_to_notify + if notifier.notification_type in notification_type_status_or_checks + ] + + all_other_notifiers = [ + notifier + for notifier in notifiers_to_notify + if notifier.notification_type not in notification_type_status_or_checks + ] + + status_or_checks_results = [ + self.notify_individual_notifier(notifier, comparison) + for notifier in status_or_checks_notifiers + ] + + if status_or_checks_results and all_other_notifiers: + # if the status/check fails, sometimes we want to add helper text to the message of the other notifications, + # to better surface that the status/check failed. + # so if there are status_and_checks_notifiers and all_other_notifiers, do the status_and_checks_notifiers first, + # look at the results of the checks, if any failed AND they are the type we have helper text for, + # add that text onto the other notifiers messages. + for result in status_or_checks_results: + notification_result = result["result"] + if ( + notification_result is not None + and notification_result.data_sent.get("state") + == StatusState.failure.value + ) and notification_result.data_sent.get("included_helper_text"): + status_or_checks_helper_text.update( + notification_result.data_sent["included_helper_text"] + ) + # TODO: pass status_or_checks_helper_text to all_other_notifiers, + # where they can integrate the helper text into their messages + results = results + status_or_checks_results + + results = results + [ + self.notify_individual_notifier(notifier, comparison) + for notifier in all_other_notifiers + ] return results def notify_individual_notifier( - self, notifier: AbstractBaseNotifier, comparison: ComparisonProxy + self, + notifier: AbstractBaseNotifier, + comparison: ComparisonProxy, ) -> IndividualResult: commit = comparison.head.commit base_commit = comparison.project_coverage_base.commit diff --git a/services/notification/notifiers/checks/patch.py b/services/notification/notifiers/checks/patch.py index 95b1d5fa6..fd086aa1b 100644 --- a/services/notification/notifiers/checks/patch.py +++ b/services/notification/notifiers/checks/patch.py @@ -1,18 +1,35 @@ +from typing import Any, TypedDict + from database.enums import Notification from services.comparison import ComparisonProxy, FilteredComparison from services.notification.notifiers.checks.base import ChecksNotifier -from services.notification.notifiers.mixins.status import StatusPatchMixin +from services.notification.notifiers.mixins.status import StatusPatchMixin, StatusState from services.yaml import read_yaml_field +class CheckOutput(TypedDict): + title: str + summary: str + annotations: list[Any] + + +class CheckResult(TypedDict): + state: StatusState + output: CheckOutput + included_helper_text: dict[str, str] + + class PatchChecksNotifier(StatusPatchMixin, ChecksNotifier): context = "patch" + notification_type_display_name = "check" @property def notification_type(self) -> Notification: return Notification.checks_patch - def build_payload(self, comparison: ComparisonProxy | FilteredComparison) -> dict: + def build_payload( + self, comparison: ComparisonProxy | FilteredComparison + ) -> CheckResult: """ This method build the paylod of the patch github checks. @@ -21,17 +38,23 @@ def build_payload(self, comparison: ComparisonProxy | FilteredComparison) -> dic """ if self.is_empty_upload(): state, message = self.get_status_check_for_empty_upload() - return { - "state": state, - "output": { - "title": "Empty Upload", - "summary": message, - }, - } - state, message = self.get_patch_status(comparison) + result = CheckResult( + state=state, + output=CheckOutput( + title="Empty Upload", + summary=message, + annotations=[], + ), + included_helper_text={}, + ) + return result + status_result = self.get_patch_status( + comparison, notification_type=self.notification_type_display_name + ) codecov_link = self.get_codecov_pr_link(comparison) - title = message + title = status_result["message"] + message = status_result["message"] should_use_upgrade = self.should_use_upgrade_decoration() if should_use_upgrade: @@ -54,23 +77,27 @@ def build_payload(self, comparison: ComparisonProxy | FilteredComparison) -> dic or should_use_upgrade or should_annotate is False ): - return { - "state": state, - "output": { - "title": f"{title}", - "summary": "\n\n".join([codecov_link, message]), - }, - } + result = CheckResult( + state=status_result["state"], + output=CheckOutput( + title=title, + summary="\n\n".join([codecov_link, message]), + annotations=[], + ), + included_helper_text=status_result["included_helper_text"], + ) + return result diff = comparison.get_diff(use_original_base=True) # TODO: Look into why the apply diff in get_patch_status is not saving state at this point comparison.head.report.apply_diff(diff) annotations = self.create_annotations(comparison, diff) - - return { - "state": state, - "output": { - "title": f"{title}", - "summary": "\n\n".join([codecov_link, message]), - "annotations": annotations, - }, - } + result = CheckResult( + state=status_result["state"], + output=CheckOutput( + title=title, + summary="\n\n".join([codecov_link, message]), + annotations=annotations, + ), + included_helper_text=status_result["included_helper_text"], + ) + return result diff --git a/services/notification/notifiers/mixins/status.py b/services/notification/notifiers/mixins/status.py index 13ec2cdb5..57c31f845 100644 --- a/services/notification/notifiers/mixins/status.py +++ b/services/notification/notifiers/mixins/status.py @@ -1,6 +1,7 @@ import logging from decimal import Decimal, InvalidOperation from enum import Enum +from typing import Literal, TypedDict from services.comparison import ComparisonProxy, FilteredComparison from services.yaml.reader import round_number @@ -13,7 +14,35 @@ class StatusState(Enum): failure = "failure" +class StatusResult(TypedDict): + """ + The mixins in this file do the calculations and decide the SuccessState for all Status and Checks Notifiers. + Checks have different fields than Statuses, so Checks are converted to the CheckResult type later. + """ + + state: Literal["success", "failure"] # StatusState values + message: str + included_helper_text: dict[str, str] + + +CUSTOM_TARGET_TEXT_PATCH_KEY = "custom_target_helper_text_patch" +CUSTOM_TARGET_TEXT_PROJECT_KEY = "custom_target_helper_text_project" +CUSTOM_TARGET_TEXT_VALUE = ( + "Your {context} {notification_type} has failed because the patch coverage is below the target coverage. " + "You can increase the patch coverage or adjust the " + "[target](https://docs.codecov.com/docs/commit-status#target) coverage." +) + + +HELPER_TEXT_MAP = { + CUSTOM_TARGET_TEXT_PATCH_KEY: CUSTOM_TARGET_TEXT_VALUE, + CUSTOM_TARGET_TEXT_PROJECT_KEY: CUSTOM_TARGET_TEXT_VALUE, +} + + class StatusPatchMixin(object): + context = "patch" + def _get_threshold(self) -> Decimal: """ Threshold can be configured by user, default is 0.0 @@ -29,7 +58,7 @@ def _get_threshold(self) -> Decimal: def _get_target( self, comparison: ComparisonProxy | FilteredComparison - ) -> Decimal | None: + ) -> tuple[Decimal | None, bool]: """ Target can be configured by user, default is auto, which is the coverage level from the base report. Target will be None if no report is found to compare against. @@ -39,6 +68,7 @@ def _get_target( target_coverage = Decimal( str(self.notifier_yaml_settings.get("target")).replace("%", "") ) + is_custom_target = True else: target_coverage = ( Decimal(comparison.project_coverage_base.report.totals.coverage) @@ -46,14 +76,16 @@ def _get_target( and comparison.project_coverage_base.report.totals.coverage is not None else None ) - return target_coverage + is_custom_target = False + return target_coverage, is_custom_target def get_patch_status( - self, comparison: ComparisonProxy | FilteredComparison - ) -> tuple[str, str]: + self, comparison: ComparisonProxy | FilteredComparison, notification_type: str + ) -> StatusResult: threshold = self._get_threshold() - target_coverage = self._get_target(comparison) + target_coverage, is_custom_target = self._get_target(comparison) totals = comparison.get_patch_totals() + included_helper_text = {} # coverage affected if totals and totals.lines > 0: @@ -84,7 +116,16 @@ def get_patch_status( message = ( f"{coverage_rounded}% of diff hit (target {target_rounded}%)" ) - return state, message + # TODO: + # if state == StatusState.failure.value and is_custom_target: + # helper_text = HELPER_TEXT_MAP[CUSTOM_TARGET_TEXT_PATCH_KEY].format( + # context=self.context, notification_type=notification_type + # ) + # included_helper_text[CUSTOM_TARGET_TEXT_PATCH_KEY] = helper_text + # message = message + " - " + helper_text + return StatusResult( + state=state, message=message, included_helper_text=included_helper_text + ) # coverage not affected if comparison.project_coverage_base.commit: @@ -94,10 +135,16 @@ def get_patch_status( ) else: description = "Coverage not affected" - return StatusState.success.value, description + return StatusResult( + state=StatusState.success.value, + message=description, + included_helper_text=included_helper_text, + ) class StatusChangesMixin(object): + context = "changes" + def is_a_change_worth_noting(self, change) -> bool: if not change.new and not change.deleted: # has totals and not -10m => 10h @@ -144,6 +191,7 @@ def get_changes_status( class StatusProjectMixin(object): DEFAULT_REMOVED_CODE_BEHAVIOR = "adjust_base" + context = "project" def _apply_removals_only_behavior( self, comparison: ComparisonProxy | FilteredComparison @@ -420,6 +468,11 @@ def _get_project_status( # use rounded numbers for messages target_rounded = round_number(self.current_yaml, target_coverage) message = f"{head_coverage_rounded}% (target {target_rounded}%)" + # TODO: + # helper_text = HELPER_TEXT_MAP[CUSTOM_TARGET_TEXT].format( + # context=self.context, notification_type=notification_type) + # included_helper_text[CUSTOM_TARGET_TEXT] = helper_text + # message = message + " - " + helper_text return state, message # use rounded numbers for messages diff --git a/services/notification/notifiers/status/patch.py b/services/notification/notifiers/status/patch.py index 003b9a2e9..10c52ba4e 100644 --- a/services/notification/notifiers/status/patch.py +++ b/services/notification/notifiers/status/patch.py @@ -1,6 +1,6 @@ from database.enums import Notification from services.comparison import ComparisonProxy, FilteredComparison -from services.notification.notifiers.mixins.status import StatusPatchMixin +from services.notification.notifiers.mixins.status import StatusPatchMixin, StatusResult from services.notification.notifiers.status.base import StatusNotifier @@ -17,18 +17,24 @@ class PatchStatusNotifier(StatusPatchMixin, StatusNotifier): """ context = "patch" + notification_type_display_name = "status" @property def notification_type(self) -> Notification: return Notification.status_patch - def build_payload(self, comparison: ComparisonProxy | FilteredComparison) -> dict: + def build_payload( + self, comparison: ComparisonProxy | FilteredComparison + ) -> StatusResult: if self.is_empty_upload(): state, message = self.get_status_check_for_empty_upload() - return {"state": state, "message": message} + result = StatusResult(state=state, message=message, included_helper_text={}) + return result - state, message = self.get_patch_status(comparison) + result = self.get_patch_status( + comparison, notification_type=self.notification_type_display_name + ) if self.should_use_upgrade_decoration(): - message = self.get_upgrade_message() + result["message"] = self.get_upgrade_message() - return {"state": state, "message": message} + return result diff --git a/services/notification/notifiers/tests/unit/test_checks.py b/services/notification/notifiers/tests/unit/test_checks.py index 5575f1d3f..0ebe70e3a 100644 --- a/services/notification/notifiers/tests/unit/test_checks.py +++ b/services/notification/notifiers/tests/unit/test_checks.py @@ -703,7 +703,9 @@ def test_build_flag_payload( "output": { "title": "66.67% of diff hit (target 50.00%)", "summary": f"[View this Pull Request on Codecov](test.example.br/gh/test_build_flag_payload/{sample_comparison.head.commit.repository.name}/pull/{sample_comparison.pull.pullid}?dropdown=coverage&src=pr&el=h1)\n\n66.67% of diff hit (target 50.00%)", + "annotations": [], }, + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -729,7 +731,9 @@ def test_build_upgrade_payload( "output": { "title": "Codecov Report", "summary": f"[View this Pull Request on Codecov](test.example.br/gh/test_build_upgrade_payload/{sample_comparison.head.commit.repository.name}/pull/{sample_comparison.pull.pullid}?dropdown=coverage&src=pr&el=h1)\n\nThe author of this PR, codecov-test-user, is not an activated member of this organization on Codecov.\nPlease [activate this user on Codecov](test.example.br/members/gh/test_build_upgrade_payload) to display a detailed status check.\nCoverage data is still being uploaded to Codecov.io for purposes of overall coverage calculations.\nPlease don't hesitate to email us at support@codecov.io with any questions.", + "annotations": [], }, + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -761,6 +765,7 @@ def test_build_default_payload( } ], }, + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result["output"]["summary"] == result["output"]["summary"] @@ -783,7 +788,9 @@ def test_build_payload_target_coverage_failure( "output": { "title": "66.67% of diff hit (target 70.00%)", "summary": f"[View this Pull Request on Codecov](test.example.br/gh/test_build_payload_target_coverage_failure/{sample_comparison.head.commit.repository.name}/pull/{sample_comparison.pull.pullid}?dropdown=coverage&src=pr&el=h1)\n\n66.67% of diff hit (target 70.00%)", + "annotations": [], }, + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -819,6 +826,7 @@ def test_build_payload_without_base_report( } ], }, + "included_helper_text": {}, } result = notifier.build_payload(comparison) assert expected_result == result @@ -856,6 +864,7 @@ def test_build_payload_target_coverage_failure_witinh_threshold( } ], }, + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result["state"] == result["state"] @@ -898,6 +907,7 @@ def test_build_payload_with_multiple_changes( } ], }, + "included_helper_text": {}, } result = notifier.build_payload(comparison_with_multiple_changes) assert expected_result["state"] == result["state"] @@ -959,7 +969,9 @@ def test_build_payload_no_diff( "output": { "title": f"Coverage not affected when comparing {base_commit.commitid[:7]}...{head_commit.commitid[:7]}", "summary": f"[View this Pull Request on Codecov](test.example.br/gh/test_build_payload_no_diff/{sample_comparison.head.commit.repository.name}/pull/{sample_comparison.pull.pullid}?dropdown=coverage&src=pr&el=h1)\n\nCoverage not affected when comparing {base_commit.commitid[:7]}...{head_commit.commitid[:7]}", + "annotations": [], }, + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert notifier.notification_type.value == "checks_patch" @@ -1052,10 +1064,6 @@ def test_notify( mock_repo_provider.get_commit_statuses.return_value = Status([]) mock_configuration.params["setup"]["codecov_dashboard_url"] = "test.example.br" comparison = sample_comparison - payload = { - "state": "success", - "output": {"title": "Codecov Report", "summary": "Summary"}, - } mock_repo_provider.create_check_run.return_value = 2234563 mock_repo_provider.update_check_run.return_value = "success" notifier = PatchChecksNotifier( @@ -1076,7 +1084,9 @@ def test_notify( "output": { "title": f"Coverage not affected when comparing {base_commit.commitid[:7]}...{head_commit.commitid[:7]}", "summary": f"[View this Pull Request on Codecov](test.example.br/gh/test_notify/{sample_comparison.head.commit.repository.name}/pull/{sample_comparison.pull.pullid}?dropdown=coverage&src=pr&el=h1)\n\nCoverage not affected when comparing {base_commit.commitid[:7]}...{head_commit.commitid[:7]}", + "annotations": [], }, + "included_helper_text": {}, "url": f"test.example.br/gh/test_notify/{sample_comparison.head.commit.repository.name}/pull/{comparison.pull.pullid}", } @@ -1105,7 +1115,9 @@ def test_notify_passing_empty_upload( "output": { "title": "Empty Upload", "summary": "Non-testable files changed.", + "annotations": [], }, + "included_helper_text": {}, "url": f"test.example.br/gh/test_notify_passing_empty_upload/{sample_comparison.head.commit.repository.name}/pull/{comparison.pull.pullid}", } @@ -1134,7 +1146,9 @@ def test_notify_failing_empty_upload( "output": { "title": "Empty Upload", "summary": "Testable files changed", + "annotations": [], }, + "included_helper_text": {}, "url": f"test.example.br/gh/test_notify_failing_empty_upload/{sample_comparison.head.commit.repository.name}/pull/{comparison.pull.pullid}", } diff --git a/services/notification/notifiers/tests/unit/test_status.py b/services/notification/notifiers/tests/unit/test_status.py index 79c47672e..f85dbdd5f 100644 --- a/services/notification/notifiers/tests/unit/test_status.py +++ b/services/notification/notifiers/tests/unit/test_status.py @@ -2072,6 +2072,7 @@ def test_build_payload( expected_result = { "message": "66.67% of diff hit (target 50.00%)", "state": "success", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2092,6 +2093,7 @@ def test_build_upgrade_payload( expected_result = { "message": "Please activate this user to display a detailed status check", "state": "success", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2111,6 +2113,7 @@ def test_build_payload_target_coverage_failure( expected_result = { "message": "66.67% of diff hit (target 70.00%)", "state": "failure", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2130,6 +2133,7 @@ def test_build_payload_not_auto_not_string( expected_result = { "message": "66.67% of diff hit (target 57.00%)", "state": "success", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2155,6 +2159,7 @@ def test_build_payload_target_coverage_failure_within_threshold( expected_result = { "message": "66.67% of diff hit (within 5.00% threshold of 70.00%)", "state": "success", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2180,6 +2185,7 @@ def test_get_patch_status_bad_threshold( expected_result = { "message": "66.67% of diff hit (target 70.00%)", "state": "failure", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2207,6 +2213,7 @@ def test_get_patch_status_bad_threshold_fixed( expected_result = { "message": "66.67% of diff hit (within 5.00% threshold of 70.00%)", "state": "success", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2257,6 +2264,7 @@ def test_build_payload_no_diff( expected_result = { "message": f"Coverage not affected when comparing {base_commit.commitid[:7]}...{head_commit.commitid[:7]}", "state": "success", + "included_helper_text": {}, } result = notifier.build_payload(sample_comparison) assert expected_result == result @@ -2306,7 +2314,11 @@ def test_build_payload_no_diff_no_base_report( current_yaml=UserYaml({}), repository_service=mock_repo_provider, ) - expected_result = {"message": "Coverage not affected", "state": "success"} + expected_result = { + "message": "Coverage not affected", + "state": "success", + "included_helper_text": {}, + } result = notifier.build_payload(comparison) assert expected_result == result @@ -2329,6 +2341,7 @@ def test_build_payload_without_base_report( expected_result = { "message": "No report found to compare against", "state": "success", + "included_helper_text": {}, } result = notifier.build_payload(comparison) assert expected_result == result @@ -2355,6 +2368,7 @@ def test_build_payload_with_multiple_changes( expected_result = { "message": "50.00% of diff hit (target 76.92%)", "state": "failure", + "included_helper_text": {}, } result = notifier.build_payload(comparison_with_multiple_changes) assert expected_result == result diff --git a/tasks/tests/integration/test_notify_task.py b/tasks/tests/integration/test_notify_task.py index 407026f53..6840b03c1 100644 --- a/tasks/tests/integration/test_notify_task.py +++ b/tasks/tests/integration/test_notify_task.py @@ -1257,9 +1257,16 @@ def test_simple_call_status_and_notifiers( assert expected["result"].data_received == actual["result"].data_received assert expected["result"] == actual["result"] assert expected == actual - assert sorted(result["notifications"], key=lambda x: x["notifier"]) == sorted( + + sorted_result = sorted(result["notifications"], key=lambda x: x["notifier"]) + sorted_expected_result = sorted( expected_result["notifications"], key=lambda x: x["notifier"] ) + assert len(sorted_result) == len(sorted_expected_result) == 7 + assert sorted_result == sorted_expected_result + + result["notifications"] = sorted_result + expected_result["notifications"] = sorted_expected_result assert result == expected_result pull = dbsession.query(Pull).filter_by(pullid=9, repoid=commit.repoid).first()