Merge pull request #79 from altor/release_0.6.2

Release 0.6.2

Author: altor

powerapi/cli/parser.py

@@ -322,13 +322,10 @@ def subparse(self, token_list):
                               parsed argument and the result of the parsing
 
         """
-        # print(vars(self.actions))
         local_result = deepcopy(self.default_values)
         if token_list == []:
             return token_list, local_result
 
-        local_result['type'] = self.name
-
         return self._parse(token_list, local_result)
 
 
@@ -453,11 +450,11 @@ def add_component_subparser(self, component_name, subparser, help_str=''):
         """
         def _action(arg, val, args, acc):
             if arg not in acc:
-                acc[arg] = []
+                acc[arg] = {}
 
             subparser = self.subparsers_group[arg].get_subparser(val)
             args, subparse_result = subparser.subparse(args)
-            acc[arg].append(subparse_result)
+            acc[arg][subparser.name] = subparse_result
             return args, acc
 
         if component_name not in self.subparsers_group:

powerapi/cli/tools.py

@@ -159,80 +159,96 @@ def parse_argv(self):
         sys.exit()
 
 
-DB_FACTORY = {
-    'mongodb': lambda db_config: MongoDB(db_config['uri'], db_config['db'], db_config['collection']),
-    'csv': lambda db_config: CsvDB(current_path=os.getcwd() if 'directory' not in db_config else db_config['directory'],
-                                   files=[] if 'files' not in db_config else db_config['files']),
-    'influxdb': lambda db_config: InfluxDB(db_config['uri'], db_config['port'], db_config['db']),
-    'opentsdb': lambda db_config: OpenTSDB(db_config['uri'], db_config['port'], db_config['metric_name']),
-}
-
-
-MODEL_FACTORY = {
-    'hwpc_report': HWPCModel(),
-    'power_report': PowerModel(),
-}
-
-
-def generate_pullers(config, report_filter):
-    # default mode if no input are specified
-    if 'input' not in config:
-        factory = DB_FACTORY['csv']
-        model = HWPCModel()
-        name = 'csv_puller'
-        db_config = {'files': ['core.csv', 'rapl.csv', 'pcu.csv']}
-        puller = PullerActor(name, factory(db_config), report_filter, model, stream_mode=config['stream'],
-                             level_logger=config['verbose'])
-        return {name: puller}
-
-    pullers = {}
-    for db_config in config['input']:
-        try:
-            factory = DB_FACTORY[db_config['type']]
-            model = MODEL_FACTORY[db_config['model']]
-            name = db_config['name']
-            puller = PullerActor(name, factory(db_config), report_filter, model, stream_mode=config['stream'],
-                                 level_logger=config['verbose'])
-            pullers[name] = puller
-        except KeyError as exn:
-            msg = 'CLI error : argument ' + exn.args[0]
-            msg += ' needed with --output ' + db_config['type']
-            print(msg, file=sys.stderr)
+class Generator:
+
+    def __init__(self, component_group_name):
+        self.component_group_name = component_group_name
+
+    def generate(self, config):
+        if self.component_group_name not in config:
+            print('CLI error : no ' + self.component_group_name + ' specified', file=sys.stderr)
             sys.exit()
 
-    return pullers
+        actors = {}
 
+        for component_name, component_config in config[self.component_group_name].items():
+            try:
+                actors[component_config['name']] = self._gen_actor(component_name, component_config, config)
+            except KeyError as exn:
+                msg = 'CLI error : argument ' + exn.args[0]
+                msg += ' needed with --output ' + component_name
+                print(msg, file=sys.stderr)
+                sys.exit()
 
-def generate_pushers(config):
-    # default mode if no output are specified
-    if 'output' not in config:
-        factory = DB_FACTORY['csv']
-        model = PowerModel()
-        name = 'csv_pusher'
-        pusher = PusherActor(name, model, factory({}), level_logger=config['verbose'])
-        return {name: pusher}
+        return actors
 
-    pushers = {}
+    def _gen_actor(self, component_name, component_config, main_config):
+        raise NotImplementedError()
 
-    for db_config in config['output']:
-        try:
-            factory = DB_FACTORY[db_config['type']]
-            model = MODEL_FACTORY[db_config['model']]
-            name = db_config['name']
-            pusher = PusherActor(name, model, factory(db_config), level_logger=config['verbose'])
 
-            pushers[name] = pusher
+class DBActorGenerator(Generator):
 
+    def __init__(self, component_group_name):
+        Generator.__init__(self, component_group_name)
+        self.model_factory = {
+            'hwpc_report': HWPCModel(),
+            'power_report': PowerModel(),
+        }
+
+        self.db_factory = {
+            'mongodb': lambda db_config: MongoDB(db_config['uri'], db_config['db'], db_config['collection']),
+            'csv': lambda db_config: CsvDB(current_path=os.getcwd() if 'directory' not in db_config else db_config['directory'],
+                                           files=[] if 'files' not in db_config else db_config['files']),
+            'influxdb': lambda db_config: InfluxDB(db_config['uri'], db_config['port'], db_config['db']),
+            'opentsdb': lambda db_config: OpenTSDB(db_config['uri'], db_config['port'], db_config['metric_name']),
+        }
+
+    def _generate_db(self, db_name, db_config, main_config):
+        try:
+            return self.db_factory[db_name](db_config)
         except KeyError as exn:
-            msg = 'CLI error : '
+            arg = exn.args[0]
 
-            if 'type' not in db_config:
-                msg += 'output type not specified'
+            # if an argument is missing, look if it exist in another component group
+            for group_name, group in main_config.items():
+                if group_name == self.component_group_name:
+                    pass
 
-            else:
-                msg += 'argument ' + exn.args[0]
-                msg += ' needed with --output ' + db_config['type']
-            print(msg, file=sys.stderr)
-            sys.exit()
+                elif not isinstance(group, dict):
+                    pass
+
+                elif db_name not in group:
+                    pass
+
+                elif arg in group[db_name]:
+                    db_config[arg] = group[db_name][arg]
+                    return self._generate_db(db_name, db_config, main_config)
+            raise exn
+
+    def _gen_actor(self, db_name, db_config, main_config):
+        db = self._generate_db(db_name, db_config, main_config)
+        model = self.model_factory[db_config['model']]
+        name = db_config['name']
+        return self._actor_factory(name, db, model, main_config['stream'], main_config['verbose'])
+
+    def _actor_factory(self, name, db, model, stream_mode, level_logger):
+        raise NotImplementedError()
+
+
+class PullerGenerator(DBActorGenerator):
+
+    def __init__(self, report_filter):
+        DBActorGenerator.__init__(self, 'input')
+        self.report_filter = report_filter
+
+    def _actor_factory(self, name, db, model, stream_mode, level_logger):
+        return PullerActor(name, db, self.report_filter, model, stream_mode, level_logger)
+
+
+class PusherGenerator(DBActorGenerator):
+
+    def __init__(self):
+        DBActorGenerator.__init__(self, 'output')
 
-    return pushers
+    def _actor_factory(self, name, db, model, stream_mode, level_logger):
+        return PusherActor(name, model, db, level_logger)

tests/unit/cli/test_parser.py

@@ -200,7 +200,7 @@ def test_subparser():
     - "" : {}
     - "-z" : UnknowArgException(z)
     - "-a" : {a: True}
-    - "-a --sub toto -b" : {a:True, sub: [{type : toto, b: True}]}
+    - "-a --sub toto -b" : {a:True, sub: {'toto' : {b: True}}}
     - "-b" : BadContextException(b, [toto])
 
     Parser description :
@@ -223,7 +223,7 @@ def test_subparser():
     check_parsing_result(parser, '-a', {'a': True})
 
     check_parsing_result(parser, '-a --sub toto -b',
-                         {'a': True, 'sub': [{'type': 'toto', 'b': True}]})
+                         {'a': True, 'sub': {'toto': { 'b': True}}})
 
     with pytest.raises(BadContextException):
         check_parsing_result(parser, '-b', None)
@@ -364,7 +364,7 @@ def test_add_component_subparser_with_two_name():
     subparser = ComponentSubParser('titi')
     subparser.add_argument('a', 'aaa', flag=True, action=store_true, default=False)
     parser.add_component_subparser('sub', subparser)
-    check_parsing_result(parser, '--sub titi -a', {'sub': [{'type': 'titi', 'aaa': True}]})
+    check_parsing_result(parser, '--sub titi -a', {'sub': {'titi': {'aaa': True}}})
 
 
 def test_parse_empty_string_default_value():
@@ -404,25 +404,24 @@ def test_component_subparser_normal_token_list(component_subparser):
     test component_subparser, parse a token list which contain only subparser
     argument [('a', '')].
 
-    must return return a  dictionary {'type': 'test', 'a':''} as parse result
+    must return return a  dictionary {'a':''} as parse result
     and a empty token list
 
     """
-    assert component_subparser.subparse([('a', '')]) == ([], {'type': 'test',
-                                                              'a': None})
+    assert component_subparser.subparse([('a', '')]) == ([], {'a': None})
 
 
 def test_component_subparser_full_token_list(component_subparser):
     """
     test component_subparser, parse a token list which contain subparser
     argument and arguments from other parser[('a', ''), ('b', '')].
 
-    must return return a  dictionary {'type': 'test', 'a':''} as parse result
+    must return return a  dictionary {'a':''} as parse result
     and a token list that contains the unparsed arguments : [('b', '')].
 
     """
     assert component_subparser.subparse([('a', ''), ('b', '')]) == ([('b', '')],
-                                                                    {'type': 'test', 'a': None})
+                                                                    {'a': None})
 
 
 def test_subparser_empty_token_list_default_value():

tests/unit/cli/test_tools.py

@@ -0,0 +1,133 @@
+"""
+Copyright (c) 2018, INRIA
+Copyright (c) 2018, University of Lille
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+
+
+import pytest
+import sys
+
+from mock import Mock
+
+from powerapi.cli.tools import PullerGenerator, PusherGenerator
+from powerapi.puller import PullerActor
+from powerapi.database import MongoDB
+
+####################
+# PULLER GENERATOR #
+####################
+def test_no_input_specified():
+    """
+    generate puller from an empty config dict
+
+    Test if the generate_puller function call sys.exit
+    """
+    args = {}
+    sys.exit = Mock()
+    sys.exit.side_effect = Exception()
+
+    generator = PullerGenerator(None)
+
+    with pytest.raises(Exception):
+        generator.generate(args)
+
+    assert sys.exit.called
+
+
+def test_generate_puller_from_simple_config():
+    """
+    generate mongodb puller from this config :
+    { 'verbose': True, 'stream': True, 'input': {'mongodb': {'model': 'hwpc_report', 'name': 'toto', 'uri': 'titi', 'db': 'tata',
+                                             'collection': 'tutu'}}}
+
+    Test if :
+      - function return a dict containing one actor, its key is 'toto'
+      - puller type is PullerActor
+      - puller name is toto
+      - puller database type is MongoDB
+      - database uri is titi
+      - database db is tata
+      - database collection is tutu
+
+    """
+    args = {'verbose': True, 'stream': True, 'input': {'mongodb': {'model': 'hwpc_report', 'name': 'toto', 'uri': 'titi',
+                                                                   'db': 'tata', 'collection': 'tutu'}}}
+    generator = PullerGenerator(None)
+    result = generator.generate(args)
+
+    assert len(result) == 1
+    assert 'toto' in result
+    puller = result['toto']
+    assert isinstance(puller, PullerActor)
+    assert puller.name == 'toto'
+
+    db = puller.state.database
+
+    assert isinstance(db, MongoDB)
+    assert db.uri == 'titi'
+    assert db.db_name == 'tata'
+    assert db.collection_name == 'tutu'
+
+
+def test_generate_puller_and_pusher_with_same_database():
+    """
+    generate mongodb puller and pusher from this config :
+    {'verbose': True, 'stream': True, 'input': {'mongodb': {'model': 'hwpc_report', 'name': 'toto', 'uri': 'titi', 'db': 'tata',
+                                                'collection': 'tutu'}},
+     'output': {'mongodb': {'model': 'hwpc_report', 'name': 'hoho', 'collection': 'tete'}}}
+    as the database have the same uri and same database name, this value are not specified on the configuration of the output
+
+    Test if :
+      - puller and pusher database uri is titi
+      - puller and pusher database db is tata
+      - puller database collection is tutu
+      - pusher database collection is tete
+
+    """
+    args = {'verbose': True, 'stream': True, 'input': {'mongodb': {'model': 'hwpc_report', 'name': 'toto', 'uri': 'titi',
+                                                                   'db': 'tata', 'collection': 'tutu'}},
+            'output': {'mongodb': {'model': 'hwpc_report', 'name': 'hoho', 'collection': 'tete'}}}
+
+    puller_generator = PullerGenerator(None)
+    result = puller_generator.generate(args)
+
+    puller = result['toto']
+    puller_db = puller.state.database
+
+    pusher_generator = PusherGenerator()
+    result = pusher_generator.generate(args)
+
+    pusher = result['hoho']
+
+    pusher_db = pusher.state.database
+
+    assert pusher_db.uri == puller_db.uri
+    assert pusher_db.db_name == puller_db.db_name
+    assert pusher_db.collection_name == 'tete'
+    assert puller_db.collection_name == 'tutu'