Skip to content

Commit

Permalink
Remove most of the "labels" concept
Browse files Browse the repository at this point in the history
  • Loading branch information
@Swatinem
Swatinem committed Oct 14, 2024
1 parent fc29e8f commit c7b8873
Show file tree
Hide file tree
Showing 8 changed files with 31 additions and 1,617 deletions.
72 changes: 3 additions & 69 deletions services/report/languages/pycoverage.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import sentry_sdk

from services.report.languages.base import BaseLanguageProcessor
from services.report.report_builder import ReportBuilderSession, SpecialLabelsEnum
from services.report.report_builder import ReportBuilderSession

COVERAGE_HIT = 1
COVERAGE_MISS = 0
Expand All @@ -16,8 +16,6 @@ def matches_content(self, content: dict, first_line: str, name: str) -> bool:
def process(
self, content: dict, report_builder_session: ReportBuilderSession
) -> None:
labels_table = LabelsTable(report_builder_session, content)

for filename, file_coverage in content["files"].items():
_file = report_builder_session.create_coverage_file(filename)
if _file is None:
Expand All @@ -28,70 +26,6 @@ def process(
] + [(COVERAGE_MISS, ln) for ln in file_coverage["missing_lines"]]
for cov, ln in lines_and_coverage:
if ln > 0:
# label_list_of_lists: list[list[str]] | list[list[int]] = []
label_list_of_lists = [
[labels_table._normalize_label(testname)]
for testname in file_coverage.get("contexts", {}).get(
str(ln), []
)
]
_file.append(
ln,
report_builder_session.create_coverage_line(
cov,
labels_list_of_lists=label_list_of_lists,
),
)
_line = report_builder_session.create_coverage_line(cov)
_file.append(ln, _line)
report_builder_session.append(_file)


class LabelsTable:
def __init__(
self, report_builder_session: ReportBuilderSession, content: dict
) -> None:
self.labels_table: dict[str, str] = {}
self.reverse_table: dict[str, int] = {}
self.are_labels_already_encoded = False

# Compressed pycoverage files will include a labels_table
if "labels_table" in content:
self.labels_table = content["labels_table"]
# We can pre-populate some of the indexes that will be used
for idx, testname in self.labels_table.items():
clean_label = self._normalize_label(testname)
report_builder_session.label_index[int(idx)] = clean_label
self.are_labels_already_encoded = True

def _normalize_label(self, testname: int | float | str) -> str:
if isinstance(testname, int) or isinstance(testname, float):
# This is from a compressed report.
# Pull label from the labels_table
# But the labels_table keys are strings, because of JSON format
testname = self.labels_table[str(testname)]
if testname == "":
return SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER.corresponding_label
return testname.split("|", 1)[0]

def _get_list_of_label_ids(
self,
current_label_idx: dict[int, str],
line_contexts: list[str | int],
) -> list[int]:
if self.are_labels_already_encoded:
# The line contexts already include indexes in the table.
# We can re-use the table and don't have to do anything with contexts.
return sorted(map(int, line_contexts))

# In this case we do need to fix the labels
label_ids_for_line = set()
for testname in line_contexts:
clean_label = self._normalize_label(testname)
if clean_label in self.reverse_table:
label_ids_for_line.add(self.reverse_table[clean_label])
else:
label_id = max([*current_label_idx.keys(), 0]) + 1
current_label_idx[label_id] = clean_label
self.reverse_table[clean_label] = label_id
label_ids_for_line.add(label_id)

return sorted(label_ids_for_line)
124 changes: 1 addition & 123 deletions services/report/report_builder.py
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import dataclasses
import logging
from enum import Enum
from typing import Any, List, Sequence
from typing import Any, Sequence

from shared.reports.resources import LineSession, Report, ReportFile, ReportLine
from shared.reports.types import CoverageDatapoint
from shared.yaml.user_yaml import UserYaml

from helpers.labels import SpecialLabelsEnum
from services.path_fixer import PathFixer
from services.yaml.reader import read_yaml_field

Expand Down Expand Up @@ -36,8 +33,6 @@ def __init__(
self.filepath = report_filepath
self._report_builder = report_builder
self._report = Report()
self.label_index = {}
self._present_labels = set()

@property
def path_fixer(self):
Expand All @@ -53,109 +48,11 @@ def get_file(self, filename: str) -> ReportFile | None:
return self._report.get(filename)

def append(self, file: ReportFile):
if file is not None:
for line_number, line in file.lines:
if line.datapoints:
for datapoint in line.datapoints:
if datapoint.label_ids:
for label in datapoint.label_ids:
self._present_labels.add(label)
return self._report.append(file)

def output_report(self) -> Report:
"""
Outputs a Report.
This function applies all the needed modifications before a report
can be output
Returns:
Report: The legacy report desired
"""
if self._present_labels:
if self._present_labels and self._present_labels == {
SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER
}:
log.warning(
"Report only has SpecialLabels. Might indicate it was not generated with contexts"
)
for file in self._report:
for line_number, line in file.lines:
self._possibly_modify_line_to_account_for_special_labels(
file, line_number, line
)
self._report._totals = None
return self._report

def _possibly_modify_line_to_account_for_special_labels(
self, file: ReportFile, line_number: int, line: ReportLine
) -> None:
"""Possibly modify the report line in the file
to account for any label in the SpecialLabelsEnum
Args:
file (ReportFile): The file we want to modify
line_number (int): The line number in case we
need to set the new line back into the files
line (ReportLine): The original line
"""
if not line.datapoints:
return

new_datapoints = [
item
for datapoint in line.datapoints
for item in self._possibly_convert_datapoints(datapoint)
]
if new_datapoints and new_datapoints != line.datapoints:
# A check to avoid unnecessary replacement
file[line_number] = dataclasses.replace(
line,
datapoints=sorted(
new_datapoints,
key=lambda x: (
x.sessionid,
x.coverage,
x.coverage_type,
),
),
)
file._totals = None

# TODO: This can be removed after label indexing is rolled out for all customers
def _possibly_convert_datapoints(
self, datapoint: CoverageDatapoint
) -> List[CoverageDatapoint]:
"""Possibly convert datapoints
The datapoint that might need to be converted
Args:
datapoint (CoverageDatapoint): The datapoint to convert
"""
if datapoint.label_ids and any(
label == SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER
for label in datapoint.label_ids
):
new_label = (
SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER.corresponding_label
)
return [
dataclasses.replace(
datapoint,
label_ids=sorted(
set(
[
label
for label in datapoint.label_ids
if label
!= SpecialLabelsEnum.CODECOV_ALL_LABELS_PLACEHOLDER
]
+ [new_label]
)
),
)
]
return [datapoint]

def create_coverage_file(
self, path: str, do_fix_path: bool = True
) -> ReportFile | None:
Expand All @@ -171,30 +68,12 @@ def create_coverage_line(
self,
coverage: int | str,
coverage_type: CoverageType | None = None,
labels_list_of_lists: list[list[str | SpecialLabelsEnum]]
| list[list[int]]
| None = None,
partials=None,
missing_branches=None,
complexity=None,
) -> ReportLine:
sessionid = self._report_builder.sessionid
coverage_type_str = coverage_type.map_to_string() if coverage_type else None
datapoints = (
[
CoverageDatapoint(
sessionid=sessionid,
coverage=coverage,
coverage_type=coverage_type_str,
label_ids=label_ids,
)
# Avoid creating datapoints that don't contain any labels
for label_ids in (labels_list_of_lists or [])
if label_ids
]
if self._report_builder.supports_labels()
else None
)
return ReportLine.create(
coverage=coverage,
type=coverage_type_str,
Expand All @@ -209,7 +88,6 @@ def create_coverage_line(
)
)
],
datapoints=datapoints,
complexity=complexity,
)

Expand Down
Loading

0 comments on commit c7b8873

Please sign in to comment.