From c6eb8cfcb6773cd26a1cf48a4e165105ae49acba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Fri, 26 Jul 2024 10:51:54 +0200 Subject: [PATCH 1/3] Add step to compare json output The json string values can contain fnmatch patterns. Adding a new step rather than using "stdout matches line by line" because json format contains a lot of brackets that would need to be escaped when the whole line is considered a regex. It would make the expected output hard to read. In addition to that the ordering can vary, this step compares ordered jsons. --- dnf-behave-tests/dnf/steps/cmd.py | 17 +++++++ dnf-behave-tests/dnf/steps/lib/json.py | 69 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 dnf-behave-tests/dnf/steps/lib/json.py diff --git a/dnf-behave-tests/dnf/steps/cmd.py b/dnf-behave-tests/dnf/steps/cmd.py index 03803ffd8..4d04e803a 100644 --- a/dnf-behave-tests/dnf/steps/cmd.py +++ b/dnf-behave-tests/dnf/steps/cmd.py @@ -9,11 +9,13 @@ import sys import time from datetime import datetime +import json from common.lib.cmd import assert_exitcode, run_in_context from common.lib.file import prepend_installroot from fixtures import start_server_based_on_type from lib.rpmdb import get_rpmdb_rpms +from lib.json import diff_json_pattern_values def get_boot_time(): @@ -483,3 +485,18 @@ def execute_transaction(goal,description): repo_sack.update_and_load_enabled_repos(True) """ execute_python_script(context, libdnf5_setup_script + context.text) + + +@behave.then("stdout json matches") +def then_json_matches(context): + """ + Compare json output from stdout with specified json. + Specified json can contain fnmatch patterns in string values. + """ + table_json = json.loads(context.text) + out_json = json.loads(context.cmd_stdout) + diffs = diff_json_pattern_values(".", table_json, out_json) + if diffs: + for diff in diffs: + print(diff) + raise AssertionError("Expected JSON doesn't match") diff --git a/dnf-behave-tests/dnf/steps/lib/json.py b/dnf-behave-tests/dnf/steps/lib/json.py new file mode 100644 index 000000000..9a841c043 --- /dev/null +++ b/dnf-behave-tests/dnf/steps/lib/json.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import +from __future__ import print_function + +from behave.formatter.ansi_escapes import escapes +import fnmatch + +def diff_recursive(parent, obj1, obj2): + problems = [] + if isinstance(obj1, list) and isinstance(obj2, list): + # The list was originally a dictionary + if (len(obj1) != 0 and isinstance(obj1[0], tuple)) or len(obj2) != 0 and isinstance(obj2[0], tuple): + keys1 = next(zip(*obj1)) if len(obj1) != 0 else [] + keys2 = next(zip(*obj2)) if len(obj2) != 0 else [] + if keys1 != keys2: + return ["Different keys in %s object: expected: %s%s%s vs actual: %s%s%s" + % (parent, escapes['passed_arg'], keys1, escapes['reset'], escapes['failed_arg'], keys2, + escapes['reset'])] + for i in range(0, len(keys1)): + problems += diff_recursive(parent + "[" + keys1[i] + "]", obj1[i][1], obj2[i][1]) + return problems + + if len(obj1) != len(obj2): + return ["Different count of elements in %s array: Expected: %s%s%s vs Actual: %s%s%s" + % (parent, escapes['passed_arg'], len(obj1), escapes['reset'], escapes['failed_arg'], len(obj2), + escapes['reset'])] + for i in range(0, len(obj1)): + problems += diff_recursive(parent + "[" + str(i) + "]", obj1[i], obj2[i]) + return problems + + else: + # The pattern is the second argument of fnmatch + if not fnmatch.fnmatch(str(obj2), str(obj1)): + return ["Different values for %s: Expected: '%s%s%s' vs Actual: '%s%s%s'" + % (parent, escapes['passed_arg'], obj1, escapes['reset'], escapes['failed_arg'], + obj2, escapes['reset'])] + return problems + + +def diff_json_pattern_values(parent, obj1, obj2): + """ + Recursively compare json objects obj1 and obj2 + String values of obj1 can contain fnmatch patterns + + Returns a list with problems + """ + + # Recursively sort lists + # (and convert dictionaries to lists of (key, value) pairs so that they're orderable) + def ordered(obj): + if isinstance(obj, list): + return sorted(ordered(x) for x in obj) + if isinstance(obj, dict): + return sorted((k, ordered(v)) for k, v in obj.items()) + else: + return obj + + try: + obj1 = ordered(obj1) + except TypeError: + raise AssertionError("Cannot sort expected json, this could be caused by different types in an array.") + + try: + obj2 = ordered(obj2) + except TypeError: + raise AssertionError("Cannot sort input json, this could be caused by different types in an array.") + + return diff_recursive(parent, obj1, obj2) From 4c8febac94011bc9afd05e5a65e0723fd5158d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Fri, 26 Jul 2024 11:11:37 +0200 Subject: [PATCH 2/3] Add test for `advisory list` subcommand with `--json` option --- dnf-behave-tests/dnf/updateinfo-json.feature | 113 +++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 dnf-behave-tests/dnf/updateinfo-json.feature diff --git a/dnf-behave-tests/dnf/updateinfo-json.feature b/dnf-behave-tests/dnf/updateinfo-json.feature new file mode 100644 index 000000000..844d22001 --- /dev/null +++ b/dnf-behave-tests/dnf/updateinfo-json.feature @@ -0,0 +1,113 @@ +@dnf5 +Feature: dnf updateinfo command with --json + + +Background: + Given I use repository "dnf-ci-fedora" + And I successfully execute dnf with args "install glibc flac" + + +Scenario: Listing available updates in json format + When I use repository "dnf-ci-fedora-updates" + And I execute dnf with args "updateinfo list --json" + Then the exit code is 0 + And stdout json matches + """ + [ + { + "name":"FEDORA-2999:002-02", + "type":"enhancement", + "severity":"Moderate", + "nevra":"flac-1.3.3-8.fc29.x86_64", + "buildtime":"2019-01-17 00:00:00" + }, + { + "name":"FEDORA-2018-318f184000", + "type":"bugfix", + "severity":"none", + "nevra":"glibc-2.28-26.fc29.x86_64", + "buildtime":"2019-01-17 00:00:00" + } + ] + """ + + +Scenario: Listing available updates referencing bugizilla in json format + When I use repository "dnf-ci-fedora-updates" + And I execute dnf with args "updateinfo list --with-bz --json" + Then the exit code is 0 + And stdout json matches + """ + [ + { + "advisory_name":"FEDORA-2018-318f184000", + "advisory_type":"bugfix", + "advisory_severity":"bugfix", + "advisory_buildtime":"2019-01-17 00:00:00", + "nevra":"glibc-2.28-26.fc29.x86_64", + "references":[ + { + "reference_id":"222", + "reference_type":"bugzilla" + } + ] + } + ] + """ + + +Scenario: Listing updates in json format (when there's nothing to report) + When I execute dnf with args "updateinfo list --json" + Then the exit code is 0 + And stdout is + """ + [] + """ + + +Scenario: Listing updates in json format with custom type and severity + Given I use repository "advisories-base" + And I execute dnf with args "install labirinto" + And I use repository "advisories-updates" + When I execute dnf with args "updateinfo list --json" + Then the exit code is 0 + And stdout json matches + """ + [ + { + "name":"FEDORA-2019-57b5902ed1", + "type":"security", + "severity":"Critical", + "nevra":"labirinto-1.56.2-6.fc30.x86_64", + "buildtime":"2019-09-15 01:34:29" + }, + { + "name":"FEDORA-2022-2222222222", + "type":"custom_type", + "severity":"custom_severity", + "nevra":"labirinto-1.56.2-6.fc30.x86_64", + "buildtime":"2019-09-15 01:34:29" + }, + { + "name":"FEDORA-2022-2222222223", + "type":"security", + "severity":"custom_severity", + "nevra":"labirinto-1.56.2-6.fc30.x86_64", + "buildtime":"2019-09-15 01:34:29" + }, + { + "name":"FEDORA-2022-2222222224", + "type":"custom_type", + "severity":"Critical", + "nevra":"labirinto-1.56.2-6.fc30.x86_64", + "buildtime":"2019-09-15 01:34:29" + }, + { + "name":"FEDORA-2019-f4eb34cf4c", + "type":"security", + "severity":"Moderate", + "nevra":"labirinto-1.56.2-1.fc30.x86_64", + "buildtime":"2019-05-12 01:21:43" + } + ] + """ From 00e023706845156134f9ca2362d35b37b7110f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Fri, 7 Feb 2025 09:45:01 +0100 Subject: [PATCH 3/3] Add tests for advisory info --json --- dnf-behave-tests/dnf/updateinfo-json.feature | 250 +++++++++++++++++++ 1 file changed, 250 insertions(+) diff --git a/dnf-behave-tests/dnf/updateinfo-json.feature b/dnf-behave-tests/dnf/updateinfo-json.feature index 844d22001..d61f54ac5 100644 --- a/dnf-behave-tests/dnf/updateinfo-json.feature +++ b/dnf-behave-tests/dnf/updateinfo-json.feature @@ -111,3 +111,253 @@ Scenario: Listing updates in json format with custom type and severity } ] """ + + +Scenario: Info about available updates in json format + When I use repository "dnf-ci-fedora-updates" + And I execute dnf with args "updateinfo info --json" + Then the exit code is 0 + And stdout json matches + """ + { + "FEDORA-2018-318f184000":{ + "Name":"FEDORA-2018-318f184000", + "Title":"glibc bug fix", + "Severity":"none", + "Type":"bugfix", + "Status":"final", + "Vendor":"secresponseteam@foo.bar", + "Issued":"2019-01-17 00:00:00", + "Description":"Fix some stuff", + "Message":"", + "Rights":"", + "references":[ + { + "Title":"222", + "Id":"222", + "Type":"bugzilla", + "Url":"https:\/\/foobar\/foobarupdate_1" + }, + { + "Title":"CVE-2999", + "Id":"2999", + "Type":"cve", + "Url":"https:\/\/foobar\/foobarupdate_1" + }, + { + "Title":"CVE-2999", + "Id":"CVE-2999", + "Type":"cve", + "Url":"https:\/\/foobar\/foobarupdate_1" + } + ], + "collections":{ + "packages":[ + "glibc-2.28-26.fc29.x86_64" + ] + } + }, + "FEDORA-2999:002-02":{ + "Name":"FEDORA-2999:002-02", + "Title":"flac enhacements", + "Severity":"Moderate", + "Type":"enhancement", + "Status":"final", + "Vendor":"secresponseteam@foo.bar", + "Issued":"2019-01-17 00:00:00", + "Description":"Enhance some stuff", + "Message":"", + "Rights":"", + "references":[ + { + "Title":"update_1", + "Id":"1", + "Type":"self", + "Url":"https:\/\/foobar\/foobarupdate_1" + } + ], + "collections":{ + "packages":[ + "flac-1.3.3-8.fc29.x86_64" + ] + } + } + } + """ + + +Scenario: Info about available updates referencing bugizilla in json format + When I use repository "dnf-ci-fedora-updates" + And I execute dnf with args "updateinfo info --with-bz --json" + Then the exit code is 0 + And stdout json matches + """ + { + "FEDORA-2018-318f184000":{ + "Name":"FEDORA-2018-318f184000", + "Title":"glibc bug fix", + "Severity":"none", + "Type":"bugfix", + "Status":"final", + "Vendor":"secresponseteam@foo.bar", + "Issued":"2019-01-17 00:00:00", + "Description":"Fix some stuff", + "Message":"", + "Rights":"", + "references":[ + { + "Title":"222", + "Id":"222", + "Type":"bugzilla", + "Url":"https:\/\/foobar\/foobarupdate_1" + }, + { + "Title":"CVE-2999", + "Id":"2999", + "Type":"cve", + "Url":"https:\/\/foobar\/foobarupdate_1" + }, + { + "Title":"CVE-2999", + "Id":"CVE-2999", + "Type":"cve", + "Url":"https:\/\/foobar\/foobarupdate_1" + } + ], + "collections":{ + "packages":[ + "glibc-2.28-26.fc29.x86_64" + ] + } + } + } + """ + + +Scenario: Info about updates in json format (when there's nothing to report) + When I execute dnf with args "updateinfo info --json" + Then the exit code is 0 + And stdout is + """ + {{}} + """ + + +Scenario: Info about updates in json format with custom type and severity + Given I use repository "advisories-base" + And I execute dnf with args "install labirinto" + And I use repository "advisories-updates" + When I execute dnf with args "updateinfo info --json" + Then the exit code is 0 + And stdout json matches + """ + { + "FEDORA-2019-f4eb34cf4c":{ + "Name":"FEDORA-2019-f4eb34cf4c", + "Title":"labirinto-1.56.2-1.fc30", + "Severity":"Moderate", + "Type":"security", + "Status":"stable", + "Vendor":"updates@fedoraproject.org", + "Issued":"2019-05-12 01:21:43", + "Description":"GNOME 3.32.2", + "Message":"", + "Rights":"Copyright (C) 2020 Red Hat, Inc. and others.", + "references":[ + { + "Title":"[[]abrt[]] epiphany: ephy_suggestion_get_unescaped_title(): epiphany-search-provider killed by SIGABRT", + "Id":"1696529", + "Type":"bugzilla", + "Url":"https:\/\/bugzilla.redhat.com\/show_bug.cgi?id=1696529" + } + ], + "collections":{ + "packages":[ + "labirinto-1.56.2-1.fc30.i686", + "labirinto-1.56.2-1.fc30.x86_64", + "labirinto-1.56.2-1.fc30.src" + ] + } + }, + "FEDORA-2019-57b5902ed1":{ + "Name":"FEDORA-2019-57b5902ed1", + "Title":"labirinto-1.56.2-6.fc30 mozjs60-60.9.0-2.fc30 polkit-0.116-2.fc30", + "Severity":"Critical", + "Type":"security", + "Status":"stable", + "Vendor":"updates@fedoraproject.org", + "Issued":"2019-09-15 01:34:29", + "Description":"mozjs60 60.9.0, including various security, stability and regression fixes from Firefox 60.9.0 ESR. For details, see https:\/\/www.mozilla.org\/en-US\/firefox\/60.9.0\/releasenotes\/", + "Message":"", + "Rights":"Copyright (C) 2020 Red Hat, Inc. and others.", + "references":[], + "collections":{ + "packages":[ + "labirinto-1.56.2-6.fc30.i686", + "labirinto-1.56.2-6.fc30.src", + "labirinto-1.56.2-6.fc30.x86_64" + ] + } + }, + "FEDORA-2022-2222222222":{ + "Name":"FEDORA-2022-2222222222", + "Title":"labirinto-1.56.2-6.fc30 mozjs60-60.9.0-2.fc30 polkit-0.116-2.fc30", + "Severity":"custom_severity", + "Type":"custom_type", + "Status":"stable", + "Vendor":"updates@fedoraproject.org", + "Issued":"2019-09-15 01:34:29", + "Description":"advisory with custom type and seveirity", + "Message":"", + "Rights":"Copyright (C) 2020 Red Hat, Inc. and others.", + "references":[], + "collections":{ + "packages":[ + "labirinto-1.56.2-6.fc30.i686", + "labirinto-1.56.2-6.fc30.src", + "labirinto-1.56.2-6.fc30.x86_64" + ] + } + }, + "FEDORA-2022-2222222223":{ + "Name":"FEDORA-2022-2222222223", + "Title":"labirinto-1.56.2-6.fc30 mozjs60-60.9.0-2.fc30 polkit-0.116-2.fc30", + "Severity":"custom_severity", + "Type":"security", + "Status":"stable", + "Vendor":"updates@fedoraproject.org", + "Issued":"2019-09-15 01:34:29", + "Description":"advisory with custom seveirity", + "Message":"", + "Rights":"Copyright (C) 2020 Red Hat, Inc. and others.", + "references":[], + "collections":{ + "packages":[ + "labirinto-1.56.2-6.fc30.i686", + "labirinto-1.56.2-6.fc30.src", + "labirinto-1.56.2-6.fc30.x86_64" + ] + } + }, + "FEDORA-2022-2222222224":{ + "Name":"FEDORA-2022-2222222224", + "Title":"labirinto-1.56.2-6.fc30 mozjs60-60.9.0-2.fc30 polkit-0.116-2.fc30", + "Severity":"Critical", + "Type":"custom_type", + "Status":"stable", + "Vendor":"updates@fedoraproject.org", + "Issued":"2019-09-15 01:34:29", + "Description":"advisory with custom type", + "Message":"", + "Rights":"Copyright (C) 2020 Red Hat, Inc. and others.", + "references":[], + "collections":{ + "packages":[ + "labirinto-1.56.2-6.fc30.i686", + "labirinto-1.56.2-6.fc30.src", + "labirinto-1.56.2-6.fc30.x86_64" + ] + } + } + } + """