Skip to content

Split skip outcome into 4 possibilites (New) #2509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pieqq wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Split skip outcome into 4 possibilites (New) #2509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ebb7519
Split skip outcome into 4 possibilites
pieqq Apr 27, 2026
9bec781
Black
pieqq Apr 27, 2026
4595fc5
Update HTML report
pieqq Apr 27, 2026
8e6f1da
Fix HTML exporter unit tests
pieqq Apr 27, 2026
de25441
Clarify skip_reason in result.py
pieqq Apr 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion checkbox-ng/checkbox_ng/urwid_ui.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,9 +493,14 @@ class RerunWidget(CategoryWidget):

section_names = {
IJobResult.OUTCOME_FAIL: _("Failed Jobs"),
IJobResult.OUTCOME_SKIP: _("Skipped Jobs"),
IJobResult.OUTCOME_SKIP: _("Manually Skipped Jobs"),
IJobResult.OUTCOME_CRASH: _("Crashed Jobs"),
IJobResult.OUTCOME_NOT_SUPPORTED: _("Jobs with failed dependencies"),
IJobResult.OUTCOME_SKIPPED_DEPENDENCY: _(
"Jobs with failed dependencies"
),
IJobResult.OUTCOME_SKIPPED_RESOURCE: _("Jobs with unmet resources"),
IJobResult.OUTCOME_SKIPPED_MANIFEST: _("Jobs with unmet manifest"),
}

def __init__(self, node):
Expand Down
14 changes: 10 additions & 4 deletions checkbox-ng/plainbox/abc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,10 +204,16 @@ class IJobResult(metaclass=ABCMeta):
# skipped it. This is typically used for a manual job that is tedious or
# was selected by accident.
OUTCOME_SKIP = "skip"
# The not supported outcome is used when a job was about to run but a
# dependency or resource requirement prevent it from running. XXX: perhaps
# this should be called "not available", not supported has the "unsupported
# code" feeling associated with it.
# The skipped outcomes are used when a job was about to run but could not
# start for various reasons (dependency failure, resource requirement, or
# manifest requirement). These outcomes replace the old OUTCOME_NOT_SUPPORTED
# to provide more granular information about why a job was skipped.
OUTCOME_SKIPPED_DEPENDENCY = "skipped-dependency"
OUTCOME_SKIPPED_RESOURCE = "skipped-resource"
OUTCOME_SKIPPED_MANIFEST = "skipped-manifest"
# /!\ The not supported outcome is DEPRECATED.
# Use OUTCOME_SKIPPED_DEPENDENCY, OUTCOME_SKIPPED_RESOURCE or
# OUTCOME_SKIPPED_MANIFEST instead.
OUTCOME_NOT_SUPPORTED = "not-supported"
# A temporary state that should be removed later on, used to indicate that
# job runner is not implemented but the job "ran" so to speak.
Expand Down
26 changes: 10 additions & 16 deletions checkbox-ng/plainbox/impl/applogic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from plainbox.abc import IJobResult
from plainbox.i18n import gettext as _
from plainbox.impl.result import MemoryJobResult
from plainbox.impl.result_utils import determine_outcome_and_skip_reason
from plainbox.impl.secure import config
from plainbox.impl.secure.qualifiers import select_units
from plainbox.impl.session import SessionManager
Expand Down Expand Up @@ -58,23 +59,16 @@ def run_job_if_possible(session, runner, config, job, update=True, ui=None):
if job_state.can_start():
job_result = runner.run_job(job, job_state, config, ui)
else:
# Set the outcome of jobs that cannot start to
# OUTCOME_NOT_SUPPORTED _except_ if any of the inhibitors point to
# a job with an OUTCOME_SKIP outcome, if that is the case mirror
# that outcome. This makes 'skip' stronger than 'not-supported'
outcome = IJobResult.OUTCOME_NOT_SUPPORTED
for inhibitor in job_state.readiness_inhibitor_list:
if inhibitor.cause != InhibitionCause.FAILED_DEP:
continue
related_job_state = session.job_state_map[inhibitor.related_job.id]
if related_job_state.result.outcome == IJobResult.OUTCOME_SKIP:
outcome = IJobResult.OUTCOME_SKIP
job_result = MemoryJobResult(
{
"outcome": outcome,
"comments": job_state.get_readiness_description(),
}
outcome, skip_reason = determine_outcome_and_skip_reason(
job_state, session.job_state_map
)
result_data = {
"outcome": outcome,
"comments": job_state.get_readiness_description(),
}
if skip_reason:
result_data["skip_reason"] = skip_reason
job_result = MemoryJobResult(result_data)
assert job_result is not None
if update:
session.update_job_result(job, job_result)
Expand Down
6 changes: 6 additions & 0 deletions checkbox-ng/plainbox/impl/exporter/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,12 @@ def get_session_data_subset(self, session_manager):
"comments"
] = job_state.result.comments

# Add skip_reason if present
if job_state.result.skip_reason:
data["result_map"][job_id][
"skip_reason"
] = job_state.result.skip_reason

Comment on lines +276 to +281
Copy link
Copy Markdown
Collaborator Author

@pieqq pieqq Apr 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've run some tests without these lines (running a test plan and exporting as text, HTML, tar.xz), and it seems to work. I'm not sure if these are needed at all (except maybe with the old XLS exporter?).

# Add Job hash if requested
if self.OPTION_WITH_JOB_HASH in self._option_list:
data["result_map"][job_id]["hash"] = job_state.job.checksum
Expand Down
18 changes: 9 additions & 9 deletions checkbox-ng/plainbox/impl/providers/exporters/data/checkbox.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,22 +210,22 @@ <h2>Tests Results</h2>
datasets: [{
data: [
{%- for outcome, total in state.get_test_outcome_stats()|dictsort %}
{%- if outcome != 'not-supported' %}
{%- if outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] %}
{{ total }},
{%- endif %}
{%- endfor %}
],
backgroundColor: [
{%- for outcome, total in state.get_test_outcome_stats()|dictsort %}
{%- if outcome != 'not-supported' %}
{%- if outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] %}
"{{ OUTCOME_METADATA_MAP[outcome].color_hex }}",
{%- endif %}
{%- endfor %}
],
}],
labels: [
{%- for outcome, total in state.get_test_outcome_stats()|dictsort %}
{%- if outcome != 'not-supported' %}
{%- if outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] %}
"{{ OUTCOME_METADATA_MAP[outcome].tr_label }}",
{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -260,7 +260,7 @@ <h2>Tests Results</h2>
return {
// Instead of `text: label,`
// We add the value to the string
text: label + " : " + value,
text: label + ": " + value,
fillStyle: fill,
strokeStyle: stroke,
lineWidth: bw,
Expand Down Expand Up @@ -301,7 +301,7 @@ <h2>Tests Results</h2>
{%- for cat_id, cat_name in category_map|dictsort(false, 'value') %}
{% set mainloop = loop %}
{%- set has_supported = [] %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome != 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- if has_supported.append(1) %}{% endif %}
{%- endfor %}
{%- if has_supported|length > 0 %}
Expand All @@ -319,7 +319,7 @@ <h3>{{ cat_name }}<span class="ui-li-count {{ OUTCOME_METADATA_MAP[category_outc
</tr>
</thead>
<tbody>
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome != 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
<tr>
<td data-filtertext="{{ job_id|strip_ns }}" style='width:35%'>{{ job_id|strip_ns }}</td>
<td style='width:10%; font-weight: bold; color: {{ job_state.result.outcome_meta().color_hex }}'>{{ job_state.result.outcome_meta().tr_label }}</td>
Expand All @@ -344,7 +344,7 @@ <h3>{{ cat_name }}<span class="ui-li-count {{ OUTCOME_METADATA_MAP[category_outc
{%- endfor %}
<p></p>
{%- set has_any_not_supported = [] %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome == 'not-supported' and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.job.plugin not in ("resource", "attachment") %}
{%- if has_any_not_supported.append(1) %}{% endif %}
{%- endfor %}
{%- if has_any_not_supported|length > 0 %}
Expand All @@ -353,7 +353,7 @@ <h2>Not Supported Tests</h2>
{%- for cat_id, cat_name in category_map|dictsort(false, 'value') %}
{% set mainloop = loop %}
{%- set has_not_supported = [] %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome == 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- if has_not_supported.append(1) %}{% endif %}
{%- endfor %}
{%- if has_not_supported|length > 0 %}
Expand All @@ -365,7 +365,7 @@ <h3>{{ cat_name }}</h3>
</tr>
</thead>
<tbody>
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome == 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
<tr>
<td data-filtertext="{{ job_id|strip_ns }}" style='width:35%'>{{ job_id|strip_ns }}</td>
<td style='width:10%; font-weight: bold; color: {{ job_state.result.outcome_meta().color_hex }}'>{{ job_state.result.outcome_meta().tr_label }}</td>
Expand Down
1 change: 1 addition & 0 deletions checkbox-ng/plainbox/impl/providers/exporters/data/checkbox.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
"category_id": "{{ job_state.effective_category_id }}",
"status": "{{ job_state.result.outcome_meta().hexr_mapping }}",
"outcome": "{{ job_state.result.outcome }}",
"skip_reason": {{ job_state.result.skip_reason | jsonify | safe }},
"comments": {{ job_state.result.comments | jsonify | safe }},
"io_log": {{ job_state.result.io_log_as_flat_text | jsonify | safe }},
"type": "test",
Expand Down
18 changes: 9 additions & 9 deletions checkbox-ng/plainbox/impl/providers/exporters/data/multi-page.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,22 +260,22 @@ <h2>Tests Results</h2>
datasets: [{
data: [
{%- for outcome, total in state.get_test_outcome_stats()|dictsort %}
{%- if outcome != 'not-supported' %}
{%- if outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] %}
{{ total }},
{%- endif %}
{%- endfor %}
],
backgroundColor: [
{%- for outcome, total in state.get_test_outcome_stats()|dictsort %}
{%- if outcome != 'not-supported' %}
{%- if outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] %}
"{{ OUTCOME_METADATA_MAP[outcome].color_hex }}",
{%- endif %}
{%- endfor %}
],
}],
labels: [
{%- for outcome, total in state.get_test_outcome_stats()|dictsort %}
{%- if outcome != 'not-supported' %}
{%- if outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] %}
"{{ OUTCOME_METADATA_MAP[outcome].tr_label }}",
{%- endif %}
{%- endfor %}
Expand Down Expand Up @@ -310,7 +310,7 @@ <h2>Tests Results</h2>
return {
// Instead of `text: label,`
// We add the value to the string
text: label + " : " + value,
text: label + ": " + value,
fillStyle: fill,
strokeStyle: stroke,
lineWidth: bw,
Expand Down Expand Up @@ -346,7 +346,7 @@ <h2>Tests Results</h2>
{%- for cat_id, cat_name in category_map|dictsort(false, 'value') %}
{% set mainloop = loop %}
{%- set has_supported = [] %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome != 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- if has_supported.append(1) %}{% endif %}
{%- endfor %}
{%- if has_supported|length > 0 %}
Expand All @@ -364,7 +364,7 @@ <h3>{{ cat_name }}<span class="ui-li-count {{ OUTCOME_METADATA_MAP[category_outc
</tr>
</thead>
<tbody>
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome != 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome != None and job_state.result.outcome not in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
<tr>
<td data-filtertext="{{ job_id|strip_ns }}" style='width:35%'>{{ job_id|strip_ns }}</td>
<td style='width:10%; font-weight: bold; color: {{ job_state.result.outcome_meta().color_hex }}'>{{ job_state.result.outcome_meta().tr_label }}</td>
Expand All @@ -389,7 +389,7 @@ <h3>{{ cat_name }}<span class="ui-li-count {{ OUTCOME_METADATA_MAP[category_outc
{%- endfor %}
<p></p>
{%- set has_any_not_supported = [] %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome == 'not-supported' and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.job.plugin not in ("resource", "attachment") %}
{%- if has_any_not_supported.append(1) %}{% endif %}
{%- endfor %}
{%- if has_any_not_supported|length > 0 %}
Expand All @@ -398,7 +398,7 @@ <h2>Not Supported Tests</h2>
{%- for cat_id, cat_name in category_map|dictsort(false, 'value') %}
{% set mainloop = loop %}
{%- set has_not_supported = [] %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome == 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- if has_not_supported.append(1) %}{% endif %}
{%- endfor %}
{%- if has_not_supported|length > 0 %}
Expand All @@ -410,7 +410,7 @@ <h3>{{ cat_name }}</h3>
</tr>
</thead>
<tbody>
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome == 'not-supported' and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
{%- for job_id, job_state in job_state_map|dictsort if job_state.result.outcome in ["not-supported", "skipped-dependency", "skipped-resource", "skipped-manifest"] and job_state.effective_category_id == cat_id and job_state.job.plugin not in ("resource", "attachment") %}
<tr>
<td data-filtertext="{{ job_id|strip_ns }}" style='width:35%'>{{ job_id|strip_ns }}</td>
<td style='width:10%; font-weight: bold; color: {{ job_state.result.outcome_meta().color_hex }}'>{{ job_state.result.outcome_meta().tr_label }}</td>
Expand Down
Loading
Loading