diff --git a/python/vyos/qat.py b/python/vyos/qat.py new file mode 100644 index 00000000000..0c172c530fb --- /dev/null +++ b/python/vyos/qat.py @@ -0,0 +1,167 @@ +# Copyright VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +""" +Intel QuickAssist Technology (QAT) device discovery. + +The QAT kernel modules are autoloaded from the PCI modalias and the device is +brought up by the driver itself. Neither of those depends on the "system +acceleration qat" configuration node, so the state of a QAT device must be read +from the kernel and can never be inferred from the configuration session. +""" + +import os + +from glob import glob + +from vyos.utils.file import read_file + +# Intel vendor ID, as it appears in /sys/bus/pci/devices/*/vendor +PCI_VENDOR_ID_INTEL = '0x8086' + +# PCI device ID -> chipset name, for the QAT drivers VyOS ships. +# Keep in sync with verify() in src/conf_mode/system_acceleration.py. +PCI_DEVICE_IDS = { + '0x0435': 'DH895', + '0x18ee': 'QAT_200XX', + '0x19e2': 'C3xx', + '0x37c8': 'C62x', + '0x37c9': 'C62xvf', + '0x6f54': 'D15xx', +} + +PCI_DEVICES_PATH = '/sys/bus/pci/devices' +DEBUGFS_PATH = '/sys/kernel/debug' +PROC_CRYPTO = '/proc/crypto' + + +def _read_sysfs(path): + return read_file(path, defaultonfailure=None) + + +def get_debugfs_path(address: str): + """Return the debugfs directory the QAT driver created for a PCI device. + + The directory is named qat__, where depends on + which driver bound the device. Match on the address so that the type does + not have to be known in advance. Returns None when no such directory + exists, which is the case while the device has not been started. + """ + found = glob(os.path.join(DEBUGFS_PATH, f'qat_*_{address}')) + return found[0] if found else None + + +def find_devices() -> list: + """Return the QAT devices present on the PCI bus. + + Each entry carries the PCI address, the chipset name, the kernel driver + bound to the device (None when no driver is bound) and the debugfs + directory created by that driver (None when the device has not started). + """ + devices = [] + + if not os.path.isdir(PCI_DEVICES_PATH): + return devices + + for address in sorted(os.listdir(PCI_DEVICES_PATH)): + device_path = os.path.join(PCI_DEVICES_PATH, address) + + if _read_sysfs(os.path.join(device_path, 'vendor')) != PCI_VENDOR_ID_INTEL: + continue + + device_id = _read_sysfs(os.path.join(device_path, 'device')) + if device_id not in PCI_DEVICE_IDS: + continue + + driver = None + driver_link = os.path.join(device_path, 'driver') + if os.path.islink(driver_link): + try: + driver = os.path.basename(os.readlink(driver_link)) + except FileNotFoundError: + # The device can be unbound between the two calls. Report it + # as unbound rather than aborting the whole scan. + pass + + devices.append( + { + 'address': address, + 'chipset': PCI_DEVICE_IDS[device_id], + 'driver': driver, + 'debugfs': get_debugfs_path(address), + } + ) + + return devices + + +# Device states reported by get_device_state() +DEVICE_STATE_NO_DRIVER = 'no driver bound' +DEVICE_STATE_NOT_STARTED = 'not started' +DEVICE_STATE_STARTED = 'started' +DEVICE_STATE_UNKNOWN = 'unknown' + + +def debugfs_mounted() -> bool: + """Return True when debugfs is mounted. + + The QAT driver reports per-device state below it. Without debugfs a bound + device cannot be told apart from a started one, so report the difference + rather than guessing. + """ + return os.path.ismount(DEBUGFS_PATH) + + +def get_device_state(device: dict) -> str: + """Return the state of a device returned by find_devices().""" + if device['driver'] is None: + return DEVICE_STATE_NO_DRIVER + if device['debugfs'] is not None: + return DEVICE_STATE_STARTED + if not debugfs_mounted(): + return DEVICE_STATE_UNKNOWN + return DEVICE_STATE_NOT_STARTED + + +def _parse_proc_crypto(data: str): + """Yield the entries of /proc/crypto as dictionaries.""" + for block in data.split('\n\n'): + entry = {} + for line in block.splitlines(): + key, separator, value = line.partition(':') + if separator: + entry[key.strip()] = value.strip() + if entry: + yield entry + + +def get_crypto_algorithms() -> list: + """Return the kernel crypto framework algorithms served by a QAT driver. + + A non-empty result means QAT is registered with the kernel crypto API, and + therefore that in-kernel users of it - IPsec ESP among them - may have + their cryptographic operations offloaded to the QAT device, whether or not + "system acceleration qat" is configured. This is what the --enable-qat-lkcf + build option of the Intel driver turns on. + """ + crypto = read_file(PROC_CRYPTO, defaultonfailure=None) + if crypto is None: + return [] + + return [ + entry + for entry in _parse_proc_crypto(crypto) + if 'qat' in entry.get('driver', '') or 'qat' in entry.get('module', '') + ] diff --git a/smoketest/scripts/system/test_qat.py b/smoketest/scripts/system/test_qat.py new file mode 100755 index 00000000000..7ebe153ee1b --- /dev/null +++ b/smoketest/scripts/system/test_qat.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# +# Copyright VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# These tests cover the QAT device discovery helpers and the option surface of +# the op-mode script. They deliberately do not require a QAT device, so they +# are meaningful on the hardware-free images used by CI. Anything that needs +# real acceleration hardware is called out in the individual test. + +import os +import re +import unittest + +from subprocess import run +from subprocess import PIPE + +from vyos import qat +from vyos.defaults import directories + +op_mode_script = os.path.join(directories['op_mode'], 'show_acceleration.py') + + +class TestQATDeviceIDs(unittest.TestCase): + def test_table_is_well_formed(self): + self.assertTrue(qat.PCI_DEVICE_IDS) + for device_id, chipset in qat.PCI_DEVICE_IDS.items(): + self.assertRegex(device_id, r'^0x[0-9a-f]{4}$') + self.assertTrue(chipset) + + def test_qat_200xx_is_supported(self): + # 8086:18ee was added to the conf-mode PCI ID list in 2020 but never to + # the op-mode one, so "set system acceleration qat" committed while + # "show system acceleration qat" reported that no device was present. + # Both now read this table, so the two can no longer disagree. + self.assertIn('0x18ee', qat.PCI_DEVICE_IDS) + + +class TestQATDeviceState(unittest.TestCase): + # get_device_state() is a pure function over a device dictionary, so every + # state can be checked without the corresponding hardware. + def test_no_driver_bound(self): + device = {'driver': None, 'debugfs': None} + self.assertEqual(qat.get_device_state(device), qat.DEVICE_STATE_NO_DRIVER) + + def test_started(self): + device = {'driver': '200xx', 'debugfs': '/sys/kernel/debug/qat_200xx_x'} + self.assertEqual(qat.get_device_state(device), qat.DEVICE_STATE_STARTED) + + def test_bound_but_not_started(self): + device = {'driver': '200xx', 'debugfs': None} + expected = ( + qat.DEVICE_STATE_NOT_STARTED + if qat.debugfs_mounted() + else qat.DEVICE_STATE_UNKNOWN + ) + self.assertEqual(qat.get_device_state(device), expected) + + def test_states_are_distinct(self): + # The whole point of the report is that a bound-but-unconfigured + # device is not indistinguishable from an absent one. + states = { + qat.DEVICE_STATE_NO_DRIVER, + qat.DEVICE_STATE_NOT_STARTED, + qat.DEVICE_STATE_STARTED, + qat.DEVICE_STATE_UNKNOWN, + } + self.assertEqual(len(states), 4) + + +class TestQATDiscovery(unittest.TestCase): + def test_find_devices(self): + # Returns an empty list on images without QAT hardware. What matters + # here is that it never raises and that entries are well formed. + for device in qat.find_devices(): + for key in ['address', 'chipset', 'driver', 'debugfs']: + self.assertIn(key, device) + self.assertIn(device['chipset'], qat.PCI_DEVICE_IDS.values()) + + def test_proc_crypto_is_parsable(self): + # Guards the parser against changes in the kernel's /proc/crypto + # layout. Every kernel registers at least one algorithm. + from vyos.utils.file import read_file + + entries = list(qat._parse_proc_crypto(read_file(qat.PROC_CRYPTO))) + self.assertTrue(entries) + for entry in entries: + self.assertIn('name', entry) + self.assertIn('driver', entry) + + def test_crypto_algorithms(self): + # Empty unless the QAT driver registered with the kernel crypto + # framework, which requires QAT hardware and an LKCF-enabled driver. + for algorithm in qat.get_crypto_algorithms(): + self.assertTrue( + 'qat' in algorithm.get('driver', '') + or 'qat' in algorithm.get('module', '') + ) + + +class TestQATOpMode(unittest.TestCase): + def test_options_use_hyphens(self): + # The op-mode definition calls this script with hyphenated options. + # An underscore in a declaration silently breaks that call site, which + # is what happened to --dev-list. + with open(op_mode_script) as f: + source = f.read() + + options = re.findall(r'add_argument\(\s*["\'](--[^"\']+)', source) + self.assertTrue(options) + for option in options: + self.assertNotIn('_', option, f'{option} must use hyphens') + + def test_dev_list_option_accepted(self): + # Regression test: this is the completion helper for + # "show system acceleration qat device ". + result = run([op_mode_script, '--dev-list'], stdout=PIPE, stderr=PIPE) + self.assertNotIn(b'unrecognized arguments', result.stderr) + + def test_no_option_prints_help(self): + result = run([op_mode_script], stdout=PIPE, stderr=PIPE) + self.assertNotEqual(result.returncode, 0) + self.assertIn(b'usage:', result.stdout + result.stderr) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/src/conf_mode/system_acceleration.py b/src/conf_mode/system_acceleration.py index 3e7a06465d7..2318152164f 100755 --- a/src/conf_mode/system_acceleration.py +++ b/src/conf_mode/system_acceleration.py @@ -15,12 +15,11 @@ # along with this program. If not, see . import os -import re from sys import exit from vyos.config import Config -from vyos.utils.process import popen +from vyos.qat import find_devices from vyos.utils.process import run from vyos import ConfigError from vyos import airbag @@ -64,21 +63,11 @@ def verify(qat): if not os.path.exists(qat_init_script): raise ConfigError('QAT init script not found') - # Check if QAT device exist - output, err = popen('lspci -nn', decode='utf-8') - if not err: - # PCI id | Chipset - # 19e2 -> C3xx - # 37c8 -> C62x - # 37c9 -> C62xvf - # 0435 -> DH895 - # 6f54 -> D15xx - # 18ee -> QAT_200XX - data = re.findall( - '(8086:19e2)|(8086:37c[8-9])|(8086:0435)|(8086:6f54)|(8086:18ee)', output) - # If QAT devices found - if not data: - raise ConfigError('No QAT acceleration device found') + # Check if QAT device exist. The list of supported PCI IDs lives in + # vyos.qat so that conf mode and op mode cannot disagree about which + # devices are supported. + if not find_devices(): + raise ConfigError('No QAT acceleration device found') def generate(qat): return diff --git a/src/op_mode/show_acceleration.py b/src/op_mode/show_acceleration.py index 05c5913569c..d65e5db9e92 100755 --- a/src/op_mode/show_acceleration.py +++ b/src/op_mode/show_acceleration.py @@ -19,43 +19,96 @@ import re import argparse +from tabulate import tabulate + +from vyos import qat from vyos.config import Config from vyos.utils.process import call from vyos.utils.process import popen +qat_init_script = '/etc/init.d/qat_service' + + def detect_qat_dev(): - output, err = popen('lspci -nn', decode='utf-8') - if not err: - data = re.findall('(8086:19e2)|(8086:37c[8-9])|(8086:0435)|(8086:6f54)', output) - # QAT devices found - if data: - return - print("\t No QAT device found") + """Exit unless a QAT device is present on the PCI bus.""" + devices = qat.find_devices() + if devices: + return devices + + print('No QAT device found') sys.exit(1) + +def is_configured(): + return Config().exists_effective('system acceleration qat') + + def show_qat_status(): - detect_qat_dev() + """Report the state of every QAT device as the kernel sees it. + + The QAT modules autoload from the PCI modalias and the device is started + by the driver, neither of which depends on 'system acceleration qat'. + Report what the kernel is actually doing, not what the configuration says + it should be doing. + """ + devices = detect_qat_dev() + + print('Intel QAT\n') + print( + tabulate( + [ + [ + device['address'], + device['chipset'], + device['driver'] if device['driver'] else 'none', + qat.get_device_state(device), + ] + for device in devices + ], + headers=['PCI address', 'Chipset', 'Driver', 'State'], + ) + ) - # Check QAT service - if not os.path.exists('/etc/init.d/qat_service'): - print("\t QAT service not installed") - sys.exit(1) + algorithms = qat.get_crypto_algorithms() + configured = is_configured() + + count = len(algorithms) + state = 'set' if configured else 'not set' + + print() + print(f'Kernel crypto framework: {count} algorithm(s) registered by QAT') + print(f"Configuration: 'system acceleration qat' is {state}") + + if algorithms and not configured: + print( + "\nWARNING: QAT is registered with the kernel crypto framework but\n" + " 'system acceleration qat' is not set. The kernel modules\n" + ' autoload from the PCI modalias and the device is started by\n' + ' the driver, both independently of the configuration, so\n' + ' cryptographic operations - IPsec ESP among them - may still\n' + ' be offloaded to the QAT device.' + ) + + # The Intel userspace tooling reports additional detail when it is present. + # It is not required for any of the above. + if os.path.exists(qat_init_script): + print() + call(f'{qat_init_script} status') - # Show QAT service - call('/etc/init.d/qat_service status') # Return QAT devices def get_qat_devices(): - data_st, err = popen('/etc/init.d/qat_service status', decode='utf-8') + data_st, err = popen(f'{qat_init_script} status', decode='utf-8') if not err: - elm_lst = re.findall('qat_dev\d', data_st) + elm_lst = re.findall(r'qat_dev\d', data_st) print('\n'.join(elm_lst)) + # Return QAT path in sysfs def get_qat_proc_path(qat_dev): q_type = "" q_bsf = "" - output, err = popen('/etc/init.d/qat_service status', decode='utf-8') + output, err = popen(f'{qat_init_script} status', decode='utf-8') if not err: # Parse QAT service output data_st = output.split("\n") @@ -69,18 +122,17 @@ def get_qat_proc_path(qat_dev): elif re.search('bsf', elm_list[elm]): q_list = elm_list[elm].split(": ") q_bsf = q_list[1] - return "/sys/kernel/debug/qat_"+q_type+"_"+q_bsf+"/" + if q_type and q_bsf: + return f'/sys/kernel/debug/qat_{q_type}_{q_bsf}/' + + print(f'Could not determine the debugfs path for {qat_dev}') + sys.exit(1) -# Check if QAT service configured -def check_qat_if_conf(): - if not Config().exists_effective('system acceleration qat'): - print("\t system acceleration qat is not configured") - sys.exit(1) parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("--hw", action="store_true", help="Show Intel QAT HW") -group.add_argument("--dev_list", action="store_true", help="Return Intel QAT devices") +group.add_argument("--dev-list", action="store_true", help="Return Intel QAT devices") group.add_argument("--flow", action="store_true", help="Show Intel QAT flows") group.add_argument("--interrupts", action="store_true", help="Show Intel QAT interrupts") group.add_argument("--status", action="store_true", help="Show Intel QAT status") @@ -92,20 +144,22 @@ def check_qat_if_conf(): if args.hw: detect_qat_dev() - # Show available Intel QAT devices - call('lspci -nn | egrep -e \'8086:37c[8-9]|8086:19e2|8086:0435|8086:6f54\'') + # Show available Intel QAT devices. The device IDs come from the single + # table in vyos.qat so that this filter cannot drift away from the one + # used for detection. + device_filter = '|'.join(f'8086:{i[2:]}' for i in sorted(qat.PCI_DEVICE_IDS)) + call(f"lspci -nn | egrep -e '{device_filter}'") elif args.flow and args.dev: - check_qat_if_conf() + detect_qat_dev() call('cat '+get_qat_proc_path(args.dev)+"fw_counters") elif args.interrupts: - check_qat_if_conf() + detect_qat_dev() # Delete _dev from args.dev call('cat /proc/interrupts | grep qat') elif args.status: - check_qat_if_conf() show_qat_status() elif args.conf and args.dev: - check_qat_if_conf() + detect_qat_dev() call('cat '+get_qat_proc_path(args.dev)+"dev_cfg") elif args.dev_list: get_qat_devices()