feat: add configuration with config file

Author: altor

powerapi/cli/__init__.py

@@ -26,5 +26,4 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
+from .config_validator import ConfigValidator

powerapi/cli/config_validator.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2018, INRIA
+# Copyright (c) 2018, University of Lille
+# All rights reserved.
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from typing import Dict
+import logging
+
+class ConfigValidator:
+    @staticmethod
+    def validate(config: Dict):
+
+        if 'verbose' not in config:
+            config['verbose'] = logging.NOTSET
+        if 'stream' not in config:
+            config['stream'] = False
+        if 'output' not in config:
+            logging.error("no output configuration found")
+            return False
+        
+        for output_type in config['output']:
+            output_config = config['output'][output_type]
+            if 'model' not in  output_config:
+                output_config['model'] = 'HWPCReport'
+            if 'name' not in  output_config:
+                output_config['name'] = 'default_pusher'
+
+        if 'input' not in config:
+            logging.error("no input configuration found")
+            return False
+
+        for input_type in config['input']:
+            input_config = config['input'][input_type]
+            if 'model' not in input_config:
+                input_config['model'] = 'PowerReport'
+            if 'name' not in input_config:
+                input_config['name'] = 'default_puller'
+        return True
+        

powerapi/cli/parser.py

@@ -474,6 +474,7 @@ def _action(arg, val, args, acc):
             args, subparse_result = subparser.subparse(args)
 
             acc[arg][subparser.name] = subparse_result
+            # acc[arg] = subparse_result
             return args, acc
 
         if component_type not in self.subparsers_group:
@@ -512,14 +513,14 @@ def _action(arg, val, args, acc):
                 raise NoNameSpecifiedForComponentException(component_type)
 
             component_name = subparse_result['name']
+            del subparse_result['name']
 
-            if subparser.name not in acc[arg]:
-                acc[arg][subparser.name] = {}
-
-            if component_name in acc[arg][subparser.name]:
+            if component_name in acc[arg]:
                 raise ComponentAlreadyExistException(component_name)
 
-            acc[arg][subparser.name][component_name] = subparse_result
+            acc[arg][component_name] = subparse_result
+            acc[arg][component_name]['type'] = subparser.name
+
             return args, acc
 
         if 'name' not in subparser.actions:

powerapi/cli/tools.py

@@ -125,7 +125,7 @@ def __init__(self):
                                      help_str='specify a database output : --db_output database_name ARG1 ARG2 ...')
 
         subparser_prom_output = ComponentSubParser('prom')
-        subparser_prom_output.add_argument('a', 'addr', help='specify server address')
+        subparser_prom_output.add_argument('u', 'uri', help='specify server uri')
         subparser_prom_output.add_argument('p', 'port', help='specify server port', type=int)
         subparser_prom_output.add_argument('M', 'metric_name', help='speify metric name')
         subparser_prom_output.add_argument('d', 'metric_description', help='specify metric description', default='energy consumption')
@@ -138,7 +138,7 @@ def __init__(self):
                                      help_str='specify a database output : --db_output database_name ARG1 ARG2 ...')
 
         subparser_direct_prom_output = ComponentSubParser('direct_prom')
-        subparser_direct_prom_output.add_argument('a', 'addr', help='specify server address')
+        subparser_direct_prom_output.add_argument('a', 'uri', help='specify server uri')
         subparser_direct_prom_output.add_argument('p', 'port', help='specify server port', type=int)
         subparser_direct_prom_output.add_argument('M', 'metric_name', help='speify metric name')
         subparser_direct_prom_output.add_argument('d', 'metric_description', help='specify metric description', default='energy consumption')
@@ -221,19 +221,19 @@ def generate(self, config):
 
         actors = {}
 
-        for component_type, components_list in config[self.component_group_name].items():
-            for component_name, component_config in components_list.items():
-                try:
-                    actors[component_name] = self._gen_actor(component_type, component_config, config)
-                except KeyError as exn:
-                    msg = 'CLI error : argument ' + exn.args[0]
-                    msg += ' needed with --output ' + component_type
-                    print(msg, file=sys.stderr)
-                    sys.exit()
+        for component_name, component_config in config[self.component_group_name].items():
+            component_type = component_config['type']
+            try:
+                actors[component_name] = self._gen_actor(component_type, component_config, config, component_name)
+            except KeyError as exn:
+                msg = 'CLI error : argument ' + exn.args[0]
+                msg += ' needed with --output ' + component_type
+                print(msg, file=sys.stderr)
+                sys.exit()
 
         return actors
 
-    def _gen_actor(self, component_name, component_config, main_config):
+    def _gen_actor(self, component_type, component_config, main_config, component_name):
         raise NotImplementedError()
 
 
@@ -289,9 +289,9 @@ def __init__(self, component_group_name):
                                            files=[] if 'files' not in db_config else db_config['files']),
             'influxdb': lambda db_config: InfluxDB(db_config['uri'], db_config['port'], db_config['db']),
             'opentsdb': lambda db_config: OpenTSDB(db_config['uri'], db_config['port'], db_config['metric_name']),
-            'prom': lambda db_config: PrometheusDB(db_config['port'], db_config['addr'], db_config['metric_name'],
+            'prom': lambda db_config: PrometheusDB(db_config['port'], db_config['uri'], db_config['metric_name'],
                                                    db_config['metric_description'], self.model_factory[db_config['model']], db_config['aggregation_period']),
-            'direct_prom': lambda db_config: DirectPrometheusDB(db_config['port'], db_config['addr'], db_config['metric_name'],
+            'direct_prom': lambda db_config: DirectPrometheusDB(db_config['port'], db_config['uri'], db_config['metric_name'],
                                                           db_config['metric_description'], self.model_factory[db_config['model']]),
             'virtiofs': lambda db_config: VirtioFSDB(db_config['vm_name_regexp'], db_config['root_directory_name'], db_config['vm_directory_name_prefix'], db_config['vm_directory_name_suffix']),
         }
@@ -320,11 +320,10 @@ def _generate_db(self, db_name, db_config, main_config):
         return self.db_factory[db_name](db_config)
 
 
-    def _gen_actor(self, db_name, db_config, main_config):
+    def _gen_actor(self, db_name, db_config, main_config, actor_name):
         db = self._generate_db(db_name, db_config, main_config)
         model = self.model_factory[db_config['model']]
-        name = db_config['name']
-        return self._actor_factory(name, db, model, main_config['stream'], main_config['verbose'])
+        return self._actor_factory(actor_name, db, model, main_config['stream'], main_config['verbose'])
 
     def _actor_factory(self, name, db, model, stream_mode, level_logger):
         raise NotImplementedError()

tests/acceptation/test_simple_architecture_csv.py

@@ -132,11 +132,12 @@ def test_run(files, supervisor):
 
     config = {'verbose': LOG_LEVEL,
               'stream': False,
-              'input': {'csv': {'puller' : {'files': FILES,
-                                  'model': 'HWPCReport',
-                                  'name': 'puller',
-                                  }}},
-              'output': {'csv': {'pusher': {'model': 'PowerReport', 'name': 'pusher', 'directory': ROOT_PATH}}}}
+              'input': {'puller' : {'type': 'csv',
+                                    'files': FILES,
+                                    'model': 'HWPCReport',
+                                    'name': 'puller',
+                                    }},
+              'output': {'pusher': {'type': 'csv', 'model': 'PowerReport', 'name': 'pusher', 'directory': ROOT_PATH}}}
 
     # Pusher
     pusher_generator = PusherGenerator()

tests/acceptation/test_simple_architecture_with_libvirt_mapper.py

@@ -106,19 +106,20 @@ def test_run(mocked_libvirt, mongo_database, supervisor):
     LOG_LEVEL = logging.DEBUG
     config = {'verbose': LOG_LEVEL,
               'stream': False,
-              'input': {'mongodb': {'puller': {'uri': MONGO_URI,
-                                               'db': MONGO_DATABASE_NAME,
-                                               'collection': MONGO_INPUT_COLLECTION_NAME,
-                                               'model': 'HWPCReport',
-                                               'name': 'puller',
-                                               }}},
-              'output': {'mongodb': {'pusher': {'uri': MONGO_URI,
-                                                'db': MONGO_DATABASE_NAME,
-                                                'collection': MONGO_OUTPUT_COLLECTION_NAME,
-                                                'model': 'PowerReport',
-                                                'name': 'pusher'}}},
-              'report_modifier': {'libvirt_mapper': {'uri': '',
-                                                     'domain_regexp': REGEXP}}}
+              'input': {'puller': {'type': 'mongodb',
+                                   'uri': MONGO_URI,
+                                   'db': MONGO_DATABASE_NAME,
+                                   'collection': MONGO_INPUT_COLLECTION_NAME,
+                                   'model': 'HWPCReport',
+                                   }},
+              'output': {'pusher': {'type': 'mongodb',
+                                    'uri': MONGO_URI,
+                                    'db': MONGO_DATABASE_NAME,
+                                    'collection': MONGO_OUTPUT_COLLECTION_NAME,
+                                    'model': 'PowerReport',
+                                    'name': 'pusher'}},
+    'report_modifier': {'libvirt_mapper': {'uri': '',
+                                           'domain_regexp': REGEXP}}}
 
     # Pusher
     pusher_generator = PusherGenerator()

tests/integration/cli/test_generate_pusher.py

@@ -76,8 +76,8 @@ def test_generate_pusher_with_new_PowerReport_model_and_send_it_powerReport_must
     """
 
     config = {'verbose': True, 'stream': False,
-              'output': {'influxdb': {'test_pusher': {'model': 'PowerReport', 'name': 'test_pusher', 'uri': INFLUX_URI, 'port': INFLUX_PORT,
-                                                      'db': INFLUX_DBNAME}}}}
+              'output': {'test_pusher': {'type': 'influxdb', 'model': 'PowerReport', 'name': 'test_pusher', 'uri': INFLUX_URI, 'port': INFLUX_PORT,
+                                                      'db': INFLUX_DBNAME}}}
 
     class PowerModelWithFormulaName(PowerModel):
 

tests/unit/cli/test_parser.py

@@ -194,15 +194,15 @@ def test_main_parser():
         check_parsing_result(parser, '-b', None)
 
 
-def test_subparser():
+def test_actor_subparser():
     """
     test to parse strings with a parser and retrieve the following results :
 
     - "" : {}
     - "-z" : UnknowArgException(z)
     - "-a" : {a: True}
     - "-a --sub toto -b" : NoNameSpecifiedForComponentException
-    - "-a --sub toto -b --name titi" : {a:True, sub: { titi: {'toto' : {b: True}}}}
+    - "-a --sub toto -b --name titi" : {a:True, sub: { titi: { 'type': 'toto', b: True}}}
     - "-b" : BadContextException(b, [toto])
 
     Parser description :
@@ -228,13 +228,13 @@ def test_subparser():
     with pytest.raises(NoNameSpecifiedForComponentException):
         check_parsing_result(parser, '-a --sub toto -b', {})
 
-    check_parsing_result(parser, '-a --sub toto -b --name titi', {'a': True, 'sub': {'toto': {'titi': {'name': 'titi', 'b': True}}}})
+    check_parsing_result(parser, '-a --sub toto -b --name titi', {'a': True, 'sub': {'titi': {'type': 'toto', 'b': True}}})
 
     with pytest.raises(BadContextException):
         check_parsing_result(parser, '-b', None)
 
 
-def test_formula_subparser():
+def test_component_subparser():
     """
     test to parse strings with a formula parser and retrieve the following results :
     - "" : {}
@@ -265,7 +265,7 @@ def test_create_two_component():
     --sub toto --name titi --sub toto -b --name tutu
 
     test if the result is :
-    {sub:{'toto' : {'titi': {'name': 'titi'}, 'tutu': {'name': 'tutu', 'b':False}}}}
+    {sub:{'titi' : {'type': 'toto'}, 'tutu': {'type': 'toto', 'b':True}}}
 
     """
     parser = MainParser(help_arg=False)
@@ -275,7 +275,28 @@ def test_create_two_component():
     subparser.add_argument('n', 'name')
     parser.add_actor_subparser('sub', subparser)
 
-    check_parsing_result(parser, '--sub toto --name titi --sub toto -b --name tutu', {'sub': {'toto': {'titi': {'name': 'titi'}, 'tutu': {'name': 'tutu', 'b': True}}}})
+    check_parsing_result(parser, '--sub toto --name titi --sub toto -b --name tutu', {'sub': {'titi': {'type': 'toto'}, 'tutu': {'type': 'toto', 'b': True}}})
+
+def test_create_two_with_different_type_component():
+    """
+    Create two component with different type with the following cli :
+    --sub toto --name titi --sub tutu --name tete
+
+    test if the result is :
+    {sub:{'titi' : {'type': 'toto'}, 'tete': {'type': 'tutu'}}}
+
+    """
+    parser = MainParser(help_arg=False)
+    
+    subparser = ComponentSubParser('toto')
+    subparser.add_argument('n', 'name')
+    parser.add_actor_subparser('sub', subparser)
+
+    subparser = ComponentSubParser('tutu')
+    subparser.add_argument('n', 'name')
+    parser.add_actor_subparser('sub', subparser)
+
+    check_parsing_result(parser, '--sub toto --name titi --sub tutu --name tete', {'sub': {'titi': {'type': 'toto'}, 'tete': {'type': 'tutu'}}})
 
 
 def test_create_component_that_already_exist():
@@ -284,8 +305,6 @@ def test_create_component_that_already_exist():
     --sub toto --name titi --sub toto --name titi
 
     test if an ComponentAlreadyExistException is raised
-
-
     """
     parser = MainParser(help_arg=False)
 
@@ -420,7 +439,7 @@ def test_add_component_subparser_that_aldready_exists():
     parser.add_component_subparser('toto', subparser)
     subparser2 = ComponentSubParser('titi')
     subparser2.add_argument('n', 'name')
-    
+
     with pytest.raises(AlreadyAddedArgumentException):
         parser.add_component_subparser('toto', subparser2)
 
@@ -435,10 +454,10 @@ def test_add_component_subparser_with_two_name():
     subparser.add_argument('a', 'aaa', flag=True, action=store_true, default=False)
     subparser.add_argument('n', 'name')
     parser.add_actor_subparser('sub', subparser)
-    check_parsing_result(parser, '--sub titi -a --name tutu', {'sub': {'titi': {'tutu': {'aaa': True, 'name': 'tutu'}}}})
+    check_parsing_result(parser, '--sub titi -a --name tutu', {'sub': {'tutu': {'aaa': True, 'type': 'titi'}}})
 
 
-def test_add_component_subparser_that_aldready_exists():
+def test_add_component_subparser_that_aldready_exists2():
     """
     Add a component_subparser with no argument 'name'
     test if a SubParserWithoutNameArgumentException is raised