Merge pull request ARM-software#238 from bjackman/lisa-test-interface-change

Executor & LisaTest interface change

Author: derkling

ipynb/profiling/kernel_functions_profiling.ipynb

@@ -34,9 +34,6 @@
     level=logging.INFO,
     datefmt='%I:%M:%S')
 
-# Add support for Test Environment configuration
-from env import TestEnv
-
 # Add JSON parsing support
 from conf import JsonConf
 
@@ -49,7 +46,7 @@
 
 class Executor():
 
-    def __init__(self, target_conf=None, tests_conf=None):
+    def __init__(self, test_env, experiments_conf):
         """
         Tests Executor
 
@@ -58,73 +55,78 @@ def __init__(self, target_conf=None, tests_conf=None):
         - a target configuration
         - a worload to execute
 
-        The executor module can be configured to run a set of workloads
-        (wloads) in each different target configuration of a specified set
-        (confs). These wloads and confs can be specified by the "tests_config"
-        input dictionary.
+        The executor module can be configured to run a set of workloads (wloads)
+        in each different target configuration of a specified set (confs). These
+        wloads and confs can be specified by the "experiments_conf" input
+        dictionary. Each (workload, conf, iteration) tuple is called an
+        "experiment".
 
         All the results generated by each experiment will be collected a result
         folder which is named according to this template:
             results/<test_id>/<wltype>:<conf>:<wload>/<run_id>
         where:
-        - <test_id> : the "tid" defined by the tests_config, or a timestamp
+        - <test_id> : the "tid" defined by the experiments_conf, or a timestamp
                       based folder in case "tid" is not specified
         - <wltype>  : the class of workload executed, e.g. rtapp or sched_perf
         - <conf>    : the identifier of one of the specified configurations
         - <wload>   : the identified of one of the specified workload
         - <run_id>  : the progressive execution number from 1 up to the
                       specified iterations
+
+        After the workloads have been run, the Executor object's `experiments`
+        attribute is a list of Experiment objects. The `out_dir` attribute of
+        these objects can be used to find the results of the experiment.
         """
 
         # Initialize globals
         self._default_cgroup = None
         self._cgroup = None
 
         # Setup test configuration
-        if isinstance(tests_conf, dict):
+        if isinstance(experiments_conf, dict):
             logging.info('%14s - Loading custom (inline) test configuration',
                     'Target')
-            self._tests_conf = tests_conf
-        elif isinstance(tests_conf, str):
+            self._experiments_conf = experiments_conf
+        elif isinstance(experiments_conf, str):
             logging.info('%14s - Loading custom (file) test configuration',
                     'Target')
-            json_conf = JsonConf(tests_conf)
-            self._tests_conf = json_conf.load()
+            json_conf = JsonConf(experiments_conf)
+            self._experiments_conf = json_conf.load()
         else:
-            raise ValueError('test_conf must be either a dictionary or a filepath')
+            raise ValueError(
+                'experiments_conf must be either a dictionary or a filepath')
 
         # Check for mandatory configurations
-        if 'confs' not in self._tests_conf or not self._tests_conf['confs']:
+        if not self._experiments_conf.get('confs', None):
             raise ValueError(
                     'Configuration error: missing \'conf\' definitions')
-        if 'wloads' not in self._tests_conf or not self._tests_conf['wloads']:
+        if not self._experiments_conf.get('wloads', None):
             raise ValueError(
                     'Configuration error: missing \'wloads\' definitions')
 
-        # Setup devlib to access the configured target
-        self.te = TestEnv(target_conf, tests_conf)
+        self.te = test_env
         self.target = self.te.target
 
-        self._iterations = self._tests_conf.get('iterations', 1)
+        self._iterations = self._experiments_conf.get('iterations', 1)
         # Compute total number of experiments
         self._exp_count = self._iterations \
-                * len(self._tests_conf['wloads']) \
-                * len(self._tests_conf['confs'])
+                * len(self._experiments_conf['wloads']) \
+                * len(self._experiments_conf['confs'])
 
         self._print_section('Executor', 'Experiments configuration')
 
         logging.info('%14s - Configured to run:', 'Executor')
 
         logging.info('%14s -   %3d target configurations:',
-                     'Executor', len(self._tests_conf['confs']))
-        target_confs = [conf['tag'] for conf in self._tests_conf['confs']]
+                     'Executor', len(self._experiments_conf['confs']))
+        target_confs = [conf['tag'] for conf in self._experiments_conf['confs']]
         target_confs = ', '.join(target_confs)
         logging.info('%14s -       %s', 'Executor', target_confs)
 
         logging.info('%14s -   %3d workloads (%d iterations each)',
-                     'Executor', len(self._tests_conf['wloads']),
+                     'Executor', len(self._experiments_conf['wloads']),
                      self._iterations)
-        wload_confs = ', '.join(self._tests_conf['wloads'])
+        wload_confs = ', '.join(self._experiments_conf['wloads'])
         logging.info('%14s -       %s', 'Executor', wload_confs)
 
         logging.info('%14s - Total: %d experiments',
@@ -140,11 +142,11 @@ def run(self):
 
         # Run all the configured experiments
         exp_idx = 0
-        for tc in self._tests_conf['confs']:
+        for tc in self._experiments_conf['confs']:
             # TARGET: configuration
             if not self._target_configure(tc):
                 continue
-            for wl_idx in self._tests_conf['wloads']:
+            for wl_idx in self._experiments_conf['wloads']:
                 # TEST: configuration
                 wload, test_dir = self._wload_init(tc, wl_idx)
                 for itr_idx in range(1, self._iterations + 1):
@@ -502,7 +504,7 @@ def _wload_init(self, tc, wl_idx):
         tc_idx = tc['tag']
 
         # Configure the test workload
-        wlspec = self._tests_conf['wloads'][wl_idx]
+        wlspec = self._experiments_conf['wloads'][wl_idx]
         wload = self._wload_conf(wl_idx, wlspec)
 
         # Keep track of platform configuration

ipynb/tutorial/04_ExecutorUsage.ipynb

@@ -24,15 +24,37 @@
 from devlib.utils.misc import memoized
 import wrapt
 
+from env import TestEnv
 from executor import Executor
 
 class LisaTest(unittest.TestCase):
-    """A base class for LISA defined tests"""
+    """
+    A base class for LISA tests
+
+    This class is intended to be subclassed in order to create automated tests
+    for LISA. It sets up the TestEnv and Executor and provides convenience
+    methods for making assertions on results.
+
+    Subclasses should provide a test_conf to configure the TestEnv and an
+    experiments_conf to configure the executor.
+
+    Tests whose behaviour is dependent on target parameters, for example
+    presence of cpufreq governors or number of CPUs, can override
+    _getExperimentsConf to generate target-dependent experiments.
+
+    Example users of this class can be found under LISA's tests/ directory.
+    """
+
+    test_conf = None
+    """Override this with a dictionary or JSON path to configure the TestEnv"""
+
+    experiments_conf = None
+    """Override this with a dictionary to configure the Executor"""
 
     @classmethod
-    def _init(cls, conf, *args, **kwargs):
+    def _init(cls, *args, **kwargs):
         """
-        Base class to run LISA test experiments
+        Set up logging and trigger running experiments
         """
 
         cls.logger = logging.getLogger('test')
@@ -41,18 +63,39 @@ def _init(cls, conf, *args, **kwargs):
             cls.logger.setLevel(kwargs['loglevel'])
             kwargs.pop('loglevel')
 
-        cls.conf = conf
-
         cls._runExperiments()
 
+    @classmethod
+    def _getTestConf(cls):
+        if cls.test_conf is None:
+            raise NotImplementedError("Override `test_conf` attribute")
+        return cls.test_conf
+
+    @classmethod
+    def _getExperimentsConf(cls, test_env):
+        """
+        Get the experiments_conf used to configure the Executor
+
+        This method receives the initialized TestEnv as a parameter, so
+        subclasses can override it to configure workloads or target confs in a
+        manner dependent on the target. If not overridden, just returns the
+        experiments_conf attribute.
+        """
+        if cls.experiments_conf is None:
+            raise NotImplementedError("Override `experiments_conf` attribute")
+        return cls.experiments_conf
+
     @classmethod
     def _runExperiments(cls):
         """
         Default experiments execution engine
         """
 
         cls.logger.info("%14s - Setup tests execution engine...", "LisaTest")
-        cls.executor = Executor(tests_conf = cls.conf);
+        test_env = TestEnv(test_conf=cls._getTestConf())
+
+        experiments_conf = cls._getExperimentsConf(test_env)
+        cls.executor = Executor(test_env, experiments_conf)
 
         # Alias executor objects to make less verbose tests code
         cls.te = cls.executor.te