Learning updated

.gitmodules

@@ -1,8 +0,0 @@
-[submodule "pythonCode/submodules/MEDimage"]
-	path = pythonCode/submodules/MEDimage
-	url = [email protected]:MahdiAll99/MEDimage.git
-	branch = dev_lab
-[submodule "pythonCode/submodules/MEDprofiles"]
-	path = pythonCode/submodules/MEDprofiles
-	url = [email protected]:MEDomics-UdeS/MEDprofiles.git
-	branch = fusion_MEDomicsLab

pythonCode/med_libs/GoExecutionScript.py

@@ -17,19 +17,27 @@ def parse_arguments() -> tuple[dict, str]:
     parser = argparse.ArgumentParser()
     parser.add_argument('--json-param', type=str, default='.')
     parser.add_argument('--id', type=str, default='.')
+    parser.add_argument('--debug', type=bool, default=False)
     args = parser.parse_args()
-    json_params = json.loads(args.json_param)
-    id_ = args.id
-    return json_params, id_
+    if not args.debug:
+        json_params = json.loads(args.json_param)
+        id_ = args.id
+        return json_params, id_
+    else:
+        with open('json_params_dict.json', 'r') as f:
+            return json.load(f), '1234-4567-debug-id'
 
 
-def get_response_from_error(e=None, toast=None):
+def get_response_from_error(e=None, toast=None) -> dict:
     """
     Gets the response from an error
 
     Args:
         e: The error
         toast: The toast message to send to the client, ignored if e is not None
+
+    Returns:
+        The response dictionary
     """
     if e is not None:
         print(e)
@@ -58,11 +66,16 @@ class GoExecutionScript(ABC):
         _id: The id of the process
     """
 
-    def __init__(self, json_params: dict, _id: str = "default_id"):
+    def __init__(self, json_params: dict, _id: str = "default_id", debug: bool = False):
         self._json_params = json_params
         self._error_handler = None
         self._progress = {"now": 0, "currentLabel": ""}
         self._id = _id
+        self._debug = debug
+        if self._debug:
+            # save json_params_dict to a file
+            with open('json_params_dict.json', 'w') as f:
+                json.dump(json_params, f, indent=4)
 
     def start(self):
         """
@@ -71,6 +84,9 @@ def start(self):
         try:
             self.push_progress()
             results = self._custom_process(self._json_params)
+            if self._debug:
+                with open("results.json", "w") as f:
+                    f.write(json.dumps(results, indent=4))
             self.send_response(results)
         except BaseException as e:
             if self._error_handler is not None:

pythonCode/med_libs/MEDml/MEDexperiment.py

@@ -68,9 +68,7 @@ def update(self, global_json_config: json = None):
         """Updates the experiment with the pipelines and the global configuration.
 
         Args:
-            pipelines (json, optional): The pipelines of the experiment. Defaults to None.
             global_json_config (json, optional): The global configuration of the experiment. Defaults to None.
-            nb_nodes (float, optional): The number of nodes in the experiment. Defaults to 0.
         """
         self.pipelines = global_json_config['pipelines']
         self.pipelines_to_execute = self.pipelines
@@ -97,14 +95,12 @@ def create_next_nodes(self, next_nodes: json, pipelines_objects: dict) -> dict:
         nodes = {}
         if next_nodes != {}:
             for current_node_id, next_nodes_id_json in next_nodes.items():
-                # if it is a create_model node, we need to point to the model node
+                # if it is a train_model node, we need to point to the model node
                 # To be consistent with the rest of the nodes,
                 # we create a new node with the same parameters but with the model id
                 tmp_subid_list = current_node_id.split('*')
                 if len(tmp_subid_list) > 1:
-                    self.global_json_config['nodes'][current_node_id] = \
-                        copy.deepcopy(
-                            self.global_json_config['nodes'][tmp_subid_list[0]])
+                    self.global_json_config['nodes'][current_node_id] = self.global_json_config['nodes'][tmp_subid_list[0]]
                     self.global_json_config['nodes'][current_node_id]['associated_id'] = tmp_subid_list[1]
                     self.global_json_config['nodes'][current_node_id]['id'] = current_node_id
                 # then, we create the node normally
@@ -113,6 +109,7 @@ def create_next_nodes(self, next_nodes: json, pipelines_objects: dict) -> dict:
                 nodes[current_node_id] = self.handle_node_creation(
                     node, pipelines_objects)
                 nodes[current_node_id]['obj'].just_run = False
+                # if the node has next nodes
                 if current_node_id in pipelines_objects:
                     nodes[current_node_id]['next_nodes'] = \
                         self.create_next_nodes(next_nodes_id_json,
@@ -234,15 +231,17 @@ def execute_next_nodes(self, prev_node: Node, next_nodes_to_execute: json, next_
         if next_nodes_to_execute != {}:
             for current_node_id, next_nodes_id_json in next_nodes_to_execute.items():
 
+                node_can_go = True
                 node_info = next_nodes[current_node_id]
+                node = node_info['obj']
                 experiment = self.copy_experiment(experiment)
                 exp_to_return = experiment
-                node = node_info['obj']
                 self._progress['currentLabel'] = node.username
                 if not node.has_run() or prev_node.has_changed():
+                    data = node.execute(experiment, **prev_node.get_info_for_next_node())
                     node_info['results'] = {
                         'prev_node_id': prev_node.id,
-                        'data': node.execute(experiment, **prev_node.get_info_for_next_node()),
+                        'data': data,
                     }
                     # Clean node return experiment
                     if "experiment" in node_info['results']['data']:
@@ -273,7 +272,7 @@ def execute_next_nodes(self, prev_node: Node, next_nodes_to_execute: json, next_
                     results=results[current_node_id]['next_nodes'],
                     experiment=exp_to_return
                 )
-                print(f'flag-{node.username}')
+                print(f'END-{node.username}')
 
     @abstractmethod
     def modify_node_info(self, node_info: dict, node: Node, experiment: dict):
@@ -378,3 +377,27 @@ def set_progress(self, now: int = -1, label: str = "same") -> None:
             label = self._progress['currentLabel']
         self._progress = {'currentLabel': label, 'now': now}
 
+    def make_save_ready(self):
+        """Makes the experiment ready to be saved.
+        """
+        self._make_save_ready_rec(self.pipelines_objects)
+
+    @abstractmethod
+    def _make_save_ready_rec(self, next_nodes: dict):
+        """
+        Recursive function that makes the experiment ready to be saved.
+        """
+        pass
+
+    def init_obj(self):
+        """
+        Initializes the experiment object (pycaret) from a path.
+        """
+        self._init_obj_rec(self.pipelines_objects)
+
+    @abstractmethod
+    def _init_obj_rec(self, next_nodes: dict):
+        """
+        Recursive function that initializes the experiment object (pycaret) from a path.
+        """
+        pass

pythonCode/med_libs/MEDml/MEDexperiment_learning.py

@@ -74,6 +74,9 @@ def create_Node(self, node_config: dict):
         elif node_type == "finalize":
             from med_libs.MEDml.nodes.Finalize import Finalize
             return Finalize(node_config['id'], self.global_json_config)
+        elif node_type == "group_models":
+            from med_libs.MEDml.nodes.GroupModels import GroupModels
+            return GroupModels(node_config['id'], self.global_json_config)
 
     def setup_dataset(self, node: Node):
         """Sets up the dataset for the experiment.\n
@@ -111,6 +114,12 @@ def setup_dataset(self, node: Node):
             elif kwargs['use_gpu'] == "False":
                 kwargs['use_gpu'] = False
 
+        if 'index' in kwargs:
+            if kwargs['index'] == "True":
+                kwargs['index'] = True
+            elif kwargs['index'] == "False":
+                kwargs['index'] = False
+
         # add the imports
         node.CodeHandler.add_import("import numpy as np")
         node.CodeHandler.add_import("import pandas as pd")
@@ -135,9 +144,16 @@ def setup_dataset(self, node: Node):
         medml_logger = MEDml_logger()
 
         # setup the experiment
-        pycaret_exp.setup(temp_df, log_experiment=medml_logger, **kwargs)
-        node.CodeHandler.add_line(
-            "code", f"pycaret_exp.setup(temp_df, {node.CodeHandler.convert_dict_to_params(kwargs)})")
+        if 'test_data' in kwargs:
+            test_data_df = pd.read_csv(kwargs['test_data']['path'])
+            node.CodeHandler.add_line("code", f"test_data_df = pd.read_csv('{kwargs['test_data']}'")
+            node.CodeHandler.add_line("code", f"pycaret_exp.setup(temp_df, test_data=test_data_df, {node.CodeHandler.convert_dict_to_params(kwargs)})")
+            del kwargs['test_data']
+            pycaret_exp.setup(temp_df, test_data=test_data_df, log_experiment=medml_logger, **kwargs)
+        else:
+            pycaret_exp.setup(temp_df, log_experiment=medml_logger, **kwargs)
+            node.CodeHandler.add_line("code", f"pycaret_exp.setup(temp_df, {node.CodeHandler.convert_dict_to_params(kwargs)})")
+
         node.CodeHandler.add_line(
             "code", f"dataset = pycaret_exp.get_config('X').join(pycaret_exp.get_config('y'))")
         dataset_metaData = {
@@ -169,3 +185,32 @@ def setup_dataset(self, node: Node):
             'df': temp_df
         }
 
+    def _make_save_ready_rec(self, next_nodes: dict):
+        for node_id, node_content in next_nodes.items():
+            saved_path = os.path.join(
+                self.global_json_config['internalPaths']['exp'], f"exp_{node_id.replace('*', '--')}.pycaretexp")
+            if 'exp_path' in node_content['experiment']:
+                saved_path = node_content['experiment']['exp_path']
+
+            data = node_content['experiment']['pycaret_exp'].data
+            self.sceneZipFile.write_to_zip(
+                custom_actions=lambda path: node_content['experiment']['pycaret_exp'].save_experiment(saved_path))
+            node_content['experiment']['exp_path'] = saved_path
+            node_content['experiment']['dataset'] = data
+            node_content['experiment']['pycaret_exp'] = None
+            self._make_save_ready_rec(node_content['next_nodes'])
+
+    def _init_obj_rec(self, next_nodes: dict):
+        for node_id, node_content in next_nodes.items():
+            data = node_content['experiment']['dataset']
+            pycaret_exp = create_pycaret_exp(
+                ml_type=self.global_json_config['MLType'])
+            saved_path = node_content['experiment']['exp_path']
+
+            def get_experiment(pycaret_exp, data, saved_path):
+                return pycaret_exp.load_experiment(saved_path, data=data)
+
+            node_content['experiment']['pycaret_exp'] = self.sceneZipFile.read_in_zip(
+                custom_actions=lambda path: get_experiment(pycaret_exp, data, saved_path))
+
+            self._init_obj_rec(node_content['next_nodes'])

pythonCode/med_libs/MEDml/nodes/Analyze.py

@@ -40,7 +40,7 @@ def _execute(self,  experiment: dict = None, **kwargs) -> json:
         """
         selection = self.config_json['data']['internal']['selection']
         print()
-        print(Fore.BLUE + "=== Analysing === " + 'paths' +
+        print(Fore.BLUE + "=== Analysing === " +
               Fore.YELLOW + f"({self.username})" + Fore.RESET)
         print(Fore.CYAN + f"Using {selection}" + Fore.RESET)
         settings = copy.deepcopy(self.settings)

pythonCode/med_libs/MEDml/nodes/GroupModels.py

@@ -0,0 +1,94 @@
+import pandas as pd
+import copy
+import numpy as np
+import json
+
+from sklearn.pipeline import Pipeline
+
+from .NodeObj import Node, format_model, NodeCodeHandler
+from typing import Union
+from colorama import Fore
+from med_libs.server_utils import go_print
+
+DATAFRAME_LIKE = Union[dict, list, tuple, np.ndarray, pd.DataFrame]
+TARGET_LIKE = Union[int, str, list, tuple, np.ndarray, pd.Series]
+
+
+class GroupModels(Node):
+    """
+    This class represents the GroupModels node.
+    """
+
+    def __init__(self, id_: int, global_config_json: json) -> None:
+        """
+        Args:
+            id_ (int): The id of the node.
+            global_config_json (json): The global config json.
+        """
+        super().__init__(id_, global_config_json)
+        self.config_json['instance'] = 0
+        # print(f"GroupModels: {json.dumps(self.global_config_json, indent=4)}")
+        self.models_list = sorted(self.config_json['associated_id'].split('.'))
+        self.config_json['cur_models_list_id'] = []
+        self.config_json['cur_models_list_obj'] = []
+        self.config_json['cur_models_list_settings'] = []
+        print(f"GroupModels: {self.models_list}")
+        print(f"{self.config_json['cur_models_list_id']}")
+
+    def _execute(self, experiment: dict = None, **kwargs) -> json:
+        """
+        This function is used to execute the node.
+        """
+        self.config_json['instance'] += 1
+        self.config_json['cur_models_list_id'] += [kwargs['id'].split('*')[0]]
+        self.config_json['cur_models_list_settings'] += [kwargs.get('settings', {})] if 'settings' in kwargs else []
+        print()
+        print(Fore.BLUE + "=== GroupModels === " + Fore.YELLOW + f"({self.username})" + Fore.RESET)
+        print(self.config_json['instance'])
+        trained_models = kwargs['models']
+        trained_models_json = {}
+
+        for model in kwargs['models']:
+            model = format_model(model)
+            print(Fore.CYAN + f"Grouping: {model.__class__.__name__}" + Fore.RESET)
+
+        trained_models_copy = trained_models.copy()
+        self.config_json['cur_models_list_obj'] += trained_models_copy
+        self._info_for_next_node = {'models': self.config_json['cur_models_list_obj'], 'id': self.id}
+        self.CodeHandler.add_line("code", f"trained_models = []")
+
+        isLast = sorted(self.config_json['cur_models_list_id']) == self.models_list or len(
+            self.config_json['cur_models_list_id']) > len(self.models_list)
+        if isLast:
+            for settings in self.config_json['cur_models_list_settings']:
+                model_string = format_model_process(settings)
+                self.CodeHandler.add_line("code", f"trained_models += {[model_str['content'] for model_str in model_string]}".replace("\"", ""))
+        return {"prev_node_complete": isLast}
+
+
+def format_model_process(settings):
+    """
+    This function is used to format the model process.
+
+    Args:
+        settings (dict): The settings.
+
+    Returns:
+        List[dict] : The formatted model process.
+    """
+    codeHandler = NodeCodeHandler()
+    codeHandler.reset()
+    settings_cp = copy.deepcopy(settings)
+    fct_type = settings_cp['fct_type']
+    del settings_cp['fct_type']
+    if fct_type == 'compare_models':
+        codeHandler.add_line(
+            "code",
+            f"pycaret_exp.compare_models({codeHandler.convert_dict_to_params(settings_cp)})")
+
+    elif fct_type == 'train_model':
+        codeHandler.add_line(
+            "code",
+            f"pycaret_exp.create_model({codeHandler.convert_dict_to_params(settings_cp)})")
+
+    return codeHandler.get_code()

pythonCode/med_libs/MEDml/nodes/ModelHandler.py

@@ -67,7 +67,9 @@ def _execute(self, experiment: dict = None, **kwargs) -> json:
                 f"trained_models = [pycaret_exp.create_model({self.CodeHandler.convert_dict_to_params(settings)})]"
             )
         trained_models_copy = trained_models.copy()
-        self._info_for_next_node = {'models': trained_models}
+        settings_for_next = copy.deepcopy(settings)
+        settings_for_next['fct_type'] = self.type
+        self._info_for_next_node = {'models': trained_models, 'id': self.id, 'settings': settings_for_next}
         for model in trained_models_copy:
             model_copy = copy.deepcopy(model)
             trained_models_json[model_copy.__class__.__name__] = model_copy.__dict__

pythonCode/med_libs/MEDml/nodes/ModelIO.py

@@ -125,6 +125,6 @@ def _execute(self, experiment: dict = None, **kwargs) -> json:
                 "code",
                 f"pycaret_exp.load_model({self.CodeHandler.convert_dict_to_params(settings_copy)})"
             )
-            self._info_for_next_node = {'models': [trained_model]}
+            self._info_for_next_node = {'models': [trained_model], 'id': self.id}
 
         return return_val

pythonCode/med_libs/MEDml/nodes/Optimize.py

@@ -42,8 +42,7 @@ def _execute(self, experiment: dict = None, **kwargs) -> json:
         if "models" in self.type:
             self.CodeHandler.add_line(
                 "code",
-                f"optimized_model = pycaret_exp.{self.type}(trained_models, {self.CodeHandler.convert_dict_to_params(settings)})",
-                1)
+                f"optimized_model = pycaret_exp.{self.type}(trained_models, {self.CodeHandler.convert_dict_to_params(settings)})", 1)
             trained_models.append(getattr(experiment['pycaret_exp'], self.type)(input_models, **settings))
         else:
             self.CodeHandler.add_line("code", f"trained_models_optimized = []")
@@ -65,12 +64,11 @@ def _execute(self, experiment: dict = None, **kwargs) -> json:
         self.CodeHandler.add_line(
             "code", f"trained_models = trained_models_optimized")
         trained_models_copy = trained_models.copy()
-        self._info_for_next_node = {'models': trained_models}
+        self._info_for_next_node = {'models': trained_models, 'id': self.id}
         for model in trained_models_copy:
             model_copy = copy.deepcopy(model)
             trained_models_json[model_copy.__class__.__name__] = model_copy.__dict__
             for key, value in model_copy.__dict__.items():
                 if isinstance(value, np.ndarray):
-                    trained_models_json[model_copy.__class__.__name__][key] = value.tolist(
-                    )
+                    trained_models_json[model_copy.__class__.__name__][key] = value.tolist()
         return trained_models_json