Added(server-info): collection of max/cur queue count info

netutils_linux_hardware/net.py

@@ -1,11 +1,8 @@
 # coding=utf-8
-import os
 
-from six import iteritems, print_
 
 from netutils_linux_hardware.grade import Grade
-from netutils_linux_hardware.interrupts import IRQQueueCounter
-from netutils_linux_hardware.parser import Parser, YAMLLike
+from netutils_linux_hardware.net_read import ReaderNet
 from netutils_linux_hardware.rate_math import extract
 from netutils_linux_hardware.subsystem import Subsystem
 
@@ -17,15 +14,16 @@ def parse(self):
         return ReaderNet(self.datadir, self.path).netdevs
 
     def rate(self):
-        return self.map(self.__netdev, 'net')
+        return self.map(self.rate_device, 'net')
 
-    def __netdev(self, netdev):
+    def rate_device(self, netdev):
         netdevinfo = extract(self.data, ['net', netdev])
-        queues = sum(
-            len(extract(netdevinfo, ['queues', x])) for x in ('rx', 'rxtx'))
+        queues = sum(len(extract(netdevinfo, ['queues', x])) for x in ('rx', 'rxtx'))
+        queues_ethtool = self.rate_queue_ethtool(netdevinfo)
         buffers = netdevinfo.get('buffers') or {}
         return self.folding.fold({
             'queues': Grade.int(queues, 2, 8),
+            'queues_ethtool': Grade.int(queues_ethtool, 2, 8),
             'driver': {
                 'mlx5_core': 10,  # 7500 mbit/s
                 'mlx4_en': 9,  # 6500 mbit/s
@@ -43,123 +41,21 @@ def __netdev(self, netdev):
             }, self.folding.DEVICE)
         }, self.folding.DEVICE)
 
-
-class ReductorMirror(Parser):
-    @staticmethod
-    def parse(text):
-        lines = dict((line.split(' ', 1)) for line in text.strip().split('\n'))
-        for netdev, conf in iteritems(lines):
-            output = dict()
-            output['conf'] = dict()
-            output['conf']['vlan'], output['conf']['ip'] = conf.split()
-            output['conf']['vlan'] = output['conf']['vlan'] != '-'
-            if output['conf']['ip'] == '-':
-                output['conf']['ip'] = ''
-            lines[netdev] = output
-        return lines
-
-
-class NetdevParser(Parser):
-    @staticmethod
-    def parse(netdev_keys):
-        """
-        :param netdev_keys: list of device names
-        :return: dict[device_name] = {'vlan': bool; 'ip': ''}
-        """
-        netdevs = dict()
-        for key in netdev_keys:
-            if key.count('.') == 1:
-                dev, _ = key.split('.')
-            elif key.count('.') == 0:
-                dev = key
-            else:
-                print_('QinQ not supported yet. Device: {0}'.format(key))
-                raise NotImplementedError
-            netdevs[dev] = dict()
-            netdevs[dev]['conf'] = {
-                'vlan': key.count('.') == 1,
-                'ip': '',
-            }
-        return netdevs
-
-
-class BridgeOutput(NetdevParser):
-    @staticmethod
-    def parse(text):
-        """ # 0 - id, 1 - dev, 2 - _, 3 - state, 4 - _, 5 - details, 6 - _, 7 - mtu, 8 - _, 9 - master
-        :param text: # 3: eth1 state DOWN : <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 master br1 state disabled
-        :return: analyzed netdevs
-        """
-        __dev__ = 1
-        netdevs_keys = [line.split()[__dev__] for line in text.strip().split('\n')]
-        return NetdevParser.parse(netdevs_keys)
-
-
-class EthtoolFiles(NetdevParser):
-    def parse_file(self, filepath, **kwargs):
-        return self.parse(os.listdir(filepath))
-
-
-class EthtoolBuffers(Parser):
-    @staticmethod
-    def parse(text):
-        buffers = [int(line.split()[1])
-                   for line in text.strip().split('\n')
-                   if line.startswith('RX:')]
-        if buffers and len(buffers) == 2:
-            return {
-                'max': buffers[0],
-                'cur': buffers[1],
-            }
-
-
-class ReaderNet(object):
-    netdevs = None
-
-    def __init__(self, datadir, path):
-        self.datadir = datadir
-        self.path = path
-        self.net_dev_list()
-
-    def net_dev_mirror_info(self):
-        return ReductorMirror().parse_file_safe(self.path('mirror_info.conf'))
-
-    def net_dev_list_bridge(self):
-        return BridgeOutput().parse_file_safe(self.path('bridge_link'))
-
-    def net_dev_list_ethtool(self):
-        return EthtoolFiles().parse_file(self.path('ethtool/i'))
-
-    def net_dev_list_buffers(self):
-        for netdev in self.netdevs:
-            buffers_path = os.path.join(self.datadir, 'ethtool/g', netdev)
-            self.netdevs[netdev]['buffers'] = EthtoolBuffers().parse_file_safe(buffers_path)
-
-    def net_dev_list_drivers(self):
-        keys_required = (
-            'driver',
-            'version',
-        )
-        for netdev in self.netdevs:
-            driverfile = os.path.join(self.datadir, 'ethtool/i', netdev)
-            driverdata = YAMLLike().parse_file_safe(driverfile)
-            if driverdata:
-                driverdata = dict((key, v) for key, v in iteritems(driverdata) if key in keys_required)
-            else:
-                driverdata = dict()
-            self.netdevs[netdev]['driver'] = driverdata
-
-    def net_dev_list(self):
-        """
-        Priority:
-        1. mirror_info.conf (collected only in case of reductor (master conf))
-        2. bridges output (collected only in case of reductor (manual conf))
-        3. ethtool information (non reductor case)
-        """
-        self.netdevs = self.net_dev_mirror_info() or self.net_dev_list_bridge() or self.net_dev_list_ethtool()
-        if not self.netdevs:
-            return
-        self.net_dev_list_buffers()
-        self.net_dev_list_drivers()
-        interrupts = self.path('interrupts')
-        IRQQueueCounter().parse_file_safe(interrupts, netdevs=self.netdevs)
+    def rate_queue_ethtool(self, netdevinfo):
+        # TODO: numa belonging
+        # import json
+        # print(json.dumps(netdevinfo, indent=2))
+        # exit(0)
+        queues_ethtool = netdevinfo['queues_ethtool']
+        queues_type = 'rx'
+        if not queues_ethtool:
+            return 1
+        if len(queues_ethtool.keys()) == 1:
+            queues_type = list(queues_ethtool.keys())[0]
+        cpu_count = len(self.data.get('cpu').get('layout'))
+        # print(json.dumps(queues_ethtool, indent=2))
+        # print(json.dumps(self.data.get('cpu').get('layout'), indent=4))
+        queues_count = 0
+        if queues_type == 'combined' or queues_type == 'rx':
+            queues_count = queues_ethtool.get(queues_type).get('cur')
+        return int(queues_count / cpu_count * 10)

netutils_linux_hardware/net_read.py

@@ -0,0 +1,154 @@
+# coding=utf-8
+import os
+
+from six import iteritems, print_
+
+from netutils_linux_hardware.interrupts import IRQQueueCounter
+from netutils_linux_hardware.parser import Parser, YAMLLike
+
+
+class ReaderNet(object):
+    """ Reading data about network from multiple sources and merging them for every network device """
+    netdevs = None
+
+    def __init__(self, datadir, path):
+        self.datadir = datadir
+        self.path = path
+        self.net_dev_list()
+
+    def max_cur_parse(self, ethtool_key, parser, output_key):
+        for netdev in self.netdevs:
+            file_path = os.path.join(self.datadir, 'ethtool/{0}'.format(ethtool_key), netdev)
+            self.netdevs[netdev][output_key] = parser().parse_file_safe(file_path)
+
+    def net_dev_list_queues(self):
+        self.max_cur_parse('l', EthtoolQueues, 'queues_ethtool')
+        IRQQueueCounter().parse_file_safe(self.path('interrupts'), netdevs=self.netdevs)
+
+    def net_dev_list_buffers(self):
+        self.max_cur_parse('g', EthtoolBuffers, 'buffers')
+
+    def net_dev_list_drivers(self):
+        keys_required = (
+            'driver',
+            'version',
+        )
+        for netdev in self.netdevs:
+            driver_file = os.path.join(self.datadir, 'ethtool/i', netdev)
+            driver_data = YAMLLike().parse_file_safe(driver_file) or dict()
+            if driver_data:
+                driver_data = dict((key, v) for key, v in iteritems(driver_data) if key in keys_required)
+            self.netdevs[netdev]['driver'] = driver_data
+
+    def net_dev_list_numa(self):
+        for netdev in self.netdevs:
+            numa_file = os.path.join(self.datadir, 'sys/class/net/', netdev, 'device/numa_node')
+            self.netdevs[netdev]['numa_node'] = int(open(numa_file).read()) if os.path.exists(numa_file) else -1
+
+    def net_dev_list(self):
+        """
+        Priority:
+        1. mirror_info.conf (collected only in case of reductor (master conf))
+        2. bridges output (collected only in case of reductor (manual conf))
+        3. ethtool information (non reductor case)
+        """
+        self.netdevs = ReductorMirror().parse_file_safe(self.path('mirror_info.conf'))
+        if not self.netdevs:
+            self.netdevs = BridgeOutput().parse_file_safe(self.path('bridge_link'))
+        if not self.netdevs:
+            self.netdevs = EthtoolFiles().parse_file(self.path('ethtool/i'))
+        if self.netdevs:
+            self.net_dev_list_buffers()
+            self.net_dev_list_drivers()
+            self.net_dev_list_queues()
+            self.net_dev_list_numa()
+
+
+class NetdevClassificator(Parser):
+    @staticmethod
+    def parse(netdev_keys):
+        """
+        :param netdev_keys: list of device names
+        :return: dict[device_name] = {'vlan': bool; 'ip': ''}
+        """
+        netdevs = dict()
+        for key in netdev_keys:
+            if key.count('.') == 1:
+                dev, _ = key.split('.')
+            elif key.count('.') == 0:
+                dev = key
+            else:
+                print_('QinQ not supported yet. Device: {0}'.format(key))
+                raise NotImplementedError
+            netdevs[dev] = dict()
+            netdevs[dev]['conf'] = {
+                'vlan': key.count('.') == 1,
+                'ip': '',
+            }
+        return netdevs
+
+
+class ReductorMirror(Parser):
+    @staticmethod
+    def parse(text):
+        lines = dict((line.split(' ', 1)) for line in text.strip().split('\n'))
+        for netdev, conf in iteritems(lines):
+            output = dict()
+            output['conf'] = dict()
+            output['conf']['vlan'], output['conf']['ip'] = conf.split()
+            output['conf']['vlan'] = output['conf']['vlan'] != '-'
+            if output['conf']['ip'] == '-':
+                output['conf']['ip'] = ''
+            lines[netdev] = output
+        return lines
+
+
+class BridgeOutput(NetdevClassificator):
+    @staticmethod
+    def parse(text):
+        """ # 0 - id, 1 - dev, 2 - _, 3 - state, 4 - _, 5 - details, 6 - _, 7 - mtu, 8 - _, 9 - master
+        :param text: # 3: eth1 state DOWN : <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 master br1 state disabled
+        :return: analyzed netdevs
+        """
+        __dev__ = 1
+        netdevs_keys = [line.split()[__dev__] for line in text.strip().split('\n')]
+        return NetdevClassificator.parse(netdevs_keys)
+
+
+class EthtoolFiles(NetdevClassificator):
+    def parse_file(self, filepath, **kwargs):
+        return self.parse(os.listdir(filepath))
+
+
+class EthtoolBuffers(Parser):
+    @staticmethod
+    def parse(text):
+        buffers = [int(line.split()[1])
+                   for line in text.strip().split('\n')
+                   if line.startswith('RX:')]
+        if buffers and len(buffers) == 2:
+            return {
+                'max': buffers[0],
+                'cur': buffers[1],
+            }
+
+
+class EthtoolQueues(Parser):
+    @staticmethod
+    def parse(text):
+        __queues = [int(line.split()[1]) for line in text.split('\n') if line.split()[1].isdigit()]
+        if len(__queues) != 8:
+            return
+        queues = {
+            'rx': {
+                'max': __queues[0],
+                'cur': __queues[4],
+            },
+            'combined': {
+                'max': __queues[3],
+                'cur': __queues[7],
+            },
+        }
+        if queues['rx']['max'] == 0 and queues['combined']['max'] != 0:
+            del queues['rx']
+        return queues

netutils_linux_hardware/server.py

@@ -54,11 +54,10 @@ def collect(self):
 
     def read(self):
         """ Parser of raw saved data info dictionary """
-        info = dict()
-        for key, subsystem in self.subsystems.items():
-            if key != 'system' and getattr(self.args, key):
-                info[key] = subsystem(datadir=self.directory).parse()
-        if not self.args.cpu and self.args.system:
+        info = self.__read()
+        # system requires cpu because virtualization data is in lscpu output
+        # net requires cpu because queue count rate depends on NUMA's core count
+        if self.args.rate and not self.args.cpu and (self.args.system or self.args.net):
             info['cpu'] = CPU(datadir=self.directory).parse()
         return info
 
@@ -79,6 +78,13 @@ def archive(self):
         os.chdir(os.path.join(self.directory, '..'))
         os.system('tar cfz {0} {1} 2>/dev/null'.format(self.tarball, self.directory))
 
+    def __read(self):
+        return dict(
+            (key, subsystem(datadir=self.directory).parse())
+            for key, subsystem in self.subsystems.items()
+            if key != 'system' and getattr(self.args, key)
+        )
+
     def __parse_args(self):
         default_directory = '/tmp/netutils_server_info/'
         self.parser.add_argument('--directory', type=str, help="Specify a data directory or a tarball",

tests/server-info-show

@@ -12,13 +12,13 @@ run_test() {
 	echo -n "# $name "
 	server-info --show --directory="$1" > $TMPDIR/output
 	if [ "${REWRITE:-0}" = '1' ]; then
-		server-info --show --directory="$1" > "$testdata/$name/expected_output"
+		server-info --show --directory="$1" > "$testdata/$name/expected_output.yml"
 	fi
-	cmp -s $1/expected_output $TMPDIR/output || rc=$?
+	cmp -s $1/expected_output.yml $TMPDIR/output || rc=$?
 	if [ "$rc" = '0' ]; then
 		echo OK
 	else
-		diff -U 5 "$testdata/$name/expected_output" "$TMPDIR/output"
+		diff -U 5 "$testdata/$name/expected_output.yml" "$TMPDIR/output"
 		echo FAIL
 	fi
 	return "$rc"
@@ -28,12 +28,11 @@ run_test() {
 retval=0
 # shellcheck disable=SC2044
 for test in $(find "$testdata" -mindepth 1 -maxdepth 1 -type d); do
-	if [ -f "$test/expected_output" ]; then
-		if ! run_test "$test"; then
-			retval=1
-			break
-		fi
-	fi
+	[ -f "$test/expected_output.yml" ] || continue
+	run_test "$test" && continue
+	[ "${REWRITE:-0}" = '1' ] && continue
+	retval=1
+	break
 done
 rm -rf $TMPDIR
 exit "$retval"