From 9896d1b1f37984acfcf64a38e4895c1e87064f2d Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Thu, 12 Mar 2026 15:13:52 +1100 Subject: [PATCH 01/15] feat(serial): add serial file --- nf_core/configs/create/serial.py | 54 ++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 nf_core/configs/create/serial.py diff --git a/nf_core/configs/create/serial.py b/nf_core/configs/create/serial.py new file mode 100644 index 0000000000..c3b035306c --- /dev/null +++ b/nf_core/configs/create/serial.py @@ -0,0 +1,54 @@ +from typing import Any, Optional + +class NextflowSerial: + def __init__(self, data_dict: Optional[dict], tab_indent: Optional[int] = 4): + self.data_dict = data_dict + + def __getitem__(self, key: Optional[Any]) -> Any: + """Returns value from data""" + return self.data_dict[key] + + def __setitem__(self, key: Optional[Any], value: Optional[any]) -> None: + """Sets value in data""" + self.data_dict[key] = value + + @staticmethod + def _stringify(data: Optional[Any]) -> str: + """Return nextflow compatible value""" + if isinstance(data, str): + return f"'{data}'" + if isinstance(data, bool): + return 'true' if data else 'false' + if isinstance(data, type(None)): + return 'null' + return str(data) # Fallback on the Python str function if no matches + + @staticmethod + def dumps(data_dict: Optional[Any], tab_indent: Optional[int], current_indent: Optional[int], end: Optional[str] = '\n') -> str: + """Recursive function to create configuration file""" + output = "" + if isinstance(data_dict, dict): + for k, v in data_dict.items(): + if isinstance(v, dict): + output += " "*current_indent + output += f"{k} {{\n" + output += NextflowSerial.dumps(v, tab_indent = tab_indent, current_indent = current_indent + tab_indent, end = end) + output += " "*current_indent + output += f"}}{end}" + elif isinstance(v, list): + output += " "*current_indent + output += f"{k} = [" + vi = [] + for i in v: + if isinstance(i, dict): + vi.append("{" + NextflowSerial.dumps(i, tab_indent, current_indent=0, end='') + "}") + continue + vi.append(NextflowSerial.dumps(i, tab_indent, current_indent=0, end = end)) + output += ", ".join(vi) + output += f"]{end}" + else: + output += " "*current_indent + output += f"{k} = {NextflowSerial._stringify(v)}{end}" + return output + else: + return NextflowSerial._stringify(data_dict) From 6cf6cbbe4749ca8c0122b9735c35f300a5b3bb4b Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Thu, 12 Mar 2026 16:24:25 +1100 Subject: [PATCH 02/15] feat(serial): add oneliner option --- nf_core/configs/create/serial.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/nf_core/configs/create/serial.py b/nf_core/configs/create/serial.py index c3b035306c..8528d07e2b 100644 --- a/nf_core/configs/create/serial.py +++ b/nf_core/configs/create/serial.py @@ -24,13 +24,13 @@ def _stringify(data: Optional[Any]) -> str: return str(data) # Fallback on the Python str function if no matches @staticmethod - def dumps(data_dict: Optional[Any], tab_indent: Optional[int], current_indent: Optional[int], end: Optional[str] = '\n') -> str: + def dumps(data_dict: Optional[Any], tab_indent: Optional[int], current_indent: Optional[int], end: Optional[str] = '\n', indent_start: Optional[bool] = True, one_line: Optional[bool] = False) -> str: """Recursive function to create configuration file""" output = "" if isinstance(data_dict, dict): for k, v in data_dict.items(): if isinstance(v, dict): - output += " "*current_indent + output += " "*current_indent*int(indent_start) output += f"{k} {{\n" output += NextflowSerial.dumps(v, tab_indent = tab_indent, current_indent = current_indent + tab_indent, end = end) output += " "*current_indent @@ -40,15 +40,19 @@ def dumps(data_dict: Optional[Any], tab_indent: Optional[int], current_indent: O output += f"{k} = [" vi = [] for i in v: + oneliner = bool(len(i.keys()) != 1) if isinstance(i, dict): - vi.append("{" + NextflowSerial.dumps(i, tab_indent, current_indent=0, end='') + "}") + vi.append("{"*oneliner + NextflowSerial.dumps(i, tab_indent, current_indent=0, end = '', indent_start = False, one_line = not oneliner) + "}"*oneliner) continue vi.append(NextflowSerial.dumps(i, tab_indent, current_indent=0, end = end)) output += ", ".join(vi) output += f"]{end}" else: - output += " "*current_indent - output += f"{k} = {NextflowSerial._stringify(v)}{end}" + output += " "*current_indent*indent_start + if one_line: + output += f"{k}: {NextflowSerial._stringify(v)}{end}" + else: + output += f"{k} = {NextflowSerial._stringify(v)}{end}" return output else: return NextflowSerial._stringify(data_dict) From c5713449bb1b95b4f7b046398c2733b92e705c93 Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Thu, 12 Mar 2026 16:59:29 +1100 Subject: [PATCH 03/15] feat(serial): add serial method to configuration file --- nf_core/configs/create/utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index 84025a7fd4..d60de946ee 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -114,6 +114,34 @@ def __init__(self, /, **data: Any) -> None: context=_init_context_var.get(), ) + def serial(self): + """Returns a dictionary of the config""" + ret = { + "params": { + "config_profile_contact": self.config_profile_contact, + "config_profile_description": self.config_profile_description, + "config_profile_url": self.config_profile_url, + "igenomes_base": self.igenomes_base + }, + "process": { + "executor": self.scheduler, + "queue": self.queue, + "resourceLimits": [ + {"cpus": self.default_process_ncpus}, + {"memory": self.default_process_memgb}, + {"time": self.default_process_hours} + ], + "scratch": self.scratch_dir, + "maxRetries": self.retries, + }, + self.container_system: { + "enabled": True, + "cacheDir": self.cachedir, + "autoMounts": True + }, + "cleanup": self.delete_work_dir + } + @field_validator("general_config_name") @classmethod def notempty(cls, v: str) -> str: From 131fb3588386f4fba2a5d54171397f907aa5ceca Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 10:37:18 +1100 Subject: [PATCH 04/15] feat(config): replace output methods wiht serial methods --- nf_core/configs/create/create.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/configs/create/create.py b/nf_core/configs/create/create.py index 6d23036803..cbb9fde32d 100644 --- a/nf_core/configs/create/create.py +++ b/nf_core/configs/create/create.py @@ -3,6 +3,7 @@ """ from nf_core.configs.create.utils import ConfigsCreateConfig, generate_config_entry +from nf_core.configs.create.serial import NextflowSerial from re import sub from pathlib import Path @@ -313,7 +314,10 @@ def write_to_file(self): else: raise ValueError(f'Invalid config type: {self.config_type}') + serial_data = NextflowSerial.dumps(self.template_config.serial_hpc()) + with open(filename, "w+") as file: ## Write params - file.write(params_section_str) - file.write(process_section_str) + file.write(serial_data) + #file.write(params_section_str) + #file.write(process_section_str) From de40267c13950de1f05d1d28546fe07e3ceb4858 Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 10:37:47 +1100 Subject: [PATCH 05/15] fix(util): rename serial to serial_hpc --- nf_core/configs/create/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index d60de946ee..08720df477 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -114,7 +114,7 @@ def __init__(self, /, **data: Any) -> None: context=_init_context_var.get(), ) - def serial(self): + def serial_hpc(self): """Returns a dictionary of the config""" ret = { "params": { From 9bd081eb05c3010d200a8395274b592152a3f961 Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 11:44:56 +1100 Subject: [PATCH 06/15] feat(serial): add option to remove quotes around strings for one-liners --- nf_core/configs/create/serial.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/configs/create/serial.py b/nf_core/configs/create/serial.py index 8528d07e2b..7f0dbe83cb 100644 --- a/nf_core/configs/create/serial.py +++ b/nf_core/configs/create/serial.py @@ -13,10 +13,11 @@ def __setitem__(self, key: Optional[Any], value: Optional[any]) -> None: self.data_dict[key] = value @staticmethod - def _stringify(data: Optional[Any]) -> str: + def _stringify(data: Optional[Any], quote: Optional[bool] = True) -> str: """Return nextflow compatible value""" + quote = "'"*quote if isinstance(data, str): - return f"'{data}'" + return f"{quote}{data}{quote}" if isinstance(data, bool): return 'true' if data else 'false' if isinstance(data, type(None)): From 0f25c5d5bc2704234bbf2fbfd59b311a8a8d72dd Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 11:45:24 +1100 Subject: [PATCH 07/15] feat(serial): set default values for dumps --- nf_core/configs/create/serial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/configs/create/serial.py b/nf_core/configs/create/serial.py index 7f0dbe83cb..1efe6bd7be 100644 --- a/nf_core/configs/create/serial.py +++ b/nf_core/configs/create/serial.py @@ -25,7 +25,7 @@ def _stringify(data: Optional[Any], quote: Optional[bool] = True) -> str: return str(data) # Fallback on the Python str function if no matches @staticmethod - def dumps(data_dict: Optional[Any], tab_indent: Optional[int], current_indent: Optional[int], end: Optional[str] = '\n', indent_start: Optional[bool] = True, one_line: Optional[bool] = False) -> str: + def dumps(data_dict: Optional[Any], tab_indent: Optional[int] = 4, current_indent: Optional[int] = 0, end: Optional[str] = '\n', indent_start: Optional[bool] = True, one_line: Optional[bool] = False) -> str: """Recursive function to create configuration file""" output = "" if isinstance(data_dict, dict): From b9f57afd5aeedd81384a3dd1161b3b32074ecbfd Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 11:45:39 +1100 Subject: [PATCH 08/15] feat(serial): remove quotes around list dicts --- nf_core/configs/create/serial.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/configs/create/serial.py b/nf_core/configs/create/serial.py index 1efe6bd7be..f1f2108e03 100644 --- a/nf_core/configs/create/serial.py +++ b/nf_core/configs/create/serial.py @@ -51,7 +51,8 @@ def dumps(data_dict: Optional[Any], tab_indent: Optional[int] = 4, current_inden else: output += " "*current_indent*indent_start if one_line: - output += f"{k}: {NextflowSerial._stringify(v)}{end}" + # False is set here to disable quotes on strings + output += f"{k}: {NextflowSerial._stringify(v, False)}{end}" else: output += f"{k} = {NextflowSerial._stringify(v)}{end}" return output From e1041cc26d03df050c006f0b50ff450d760083ea Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 11:46:06 +1100 Subject: [PATCH 09/15] fix(serial): output formatted resource information --- nf_core/configs/create/utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index 6ae71dba65..2b4598d58b 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -131,18 +131,18 @@ def serial_hpc(self): "config_profile_contact": self.config_profile_contact, "config_profile_description": self.config_profile_description, "config_profile_url": self.config_profile_url, - "igenomes_base": self.igenomes_base + "igenomes_base": self.igenomes_cachedir }, "process": { "executor": self.scheduler, "queue": self.queue, "resourceLimits": [ - {"cpus": self.default_process_ncpus}, - {"memory": self.default_process_memgb}, - {"time": self.default_process_hours} + {"cpus": int(self.cpus)}, + {"memory": self.memory + "GB"}, + {"time": str(float(self.time)) + "h"} ], "scratch": self.scratch_dir, - "maxRetries": self.retries, + "maxRetries": int(self.retries), }, self.container_system: { "enabled": True, @@ -152,6 +152,8 @@ def serial_hpc(self): "cleanup": self.delete_work_dir } + return ret + @field_validator("general_config_name", "config_profile_description") @classmethod def notempty(cls, v: str) -> str: From b00f35082b667026e6aa4e059cfe8a5f82ababc8 Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 11:55:49 +1100 Subject: [PATCH 10/15] feat(serial): add base pipeline seiral method --- nf_core/configs/create/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index 2b4598d58b..227487d6a8 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -154,6 +154,17 @@ def serial_hpc(self): return ret + def serial_pipeline(self): + """Returns a dictionary of the pipeline config""" + ret = { + "params": { + "config_profile_contact": self.config_profile_contact, + "config_profile_description": self.config_profile_description, + "config_profile_url": self.config_profile_url, + "igenomes_base": self.igenomes_cachedir + } + } + @field_validator("general_config_name", "config_profile_description") @classmethod def notempty(cls, v: str) -> str: From 1d1fc9ea258f254d1fd991d14cb17e3a37b6acbd Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 12:43:19 +1100 Subject: [PATCH 11/15] feat(utils): add methods --- nf_core/configs/create/utils.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index 227487d6a8..fa30467d2e 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -158,12 +158,28 @@ def serial_pipeline(self): """Returns a dictionary of the pipeline config""" ret = { "params": { - "config_profile_contact": self.config_profile_contact, "config_profile_description": self.config_profile_description, - "config_profile_url": self.config_profile_url, - "igenomes_base": self.igenomes_cachedir + }, + "process": { + "cpus": self.default_process_ncpus, + "memory": self.default_process_memgb, + "time": self.default_process_hours } } + + for named in self.named_process_resources.keys(): + ret[named] = self.named_process_resources[named] + + for labelled in self.labelled_process_resources.keys(): + ret[labelled] = self.labelled_process_resources[labelled] + + return ret + + def serial(self): + if self.general_config_type != "pipeline": + return self.serial_hpc() + else: + return self.serial_pipeline() @field_validator("general_config_name", "config_profile_description") @classmethod From e96c8dcb835f12e295c2efab47f3d70e23540821 Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 13:44:26 +1100 Subject: [PATCH 12/15] revert(create): revert method call to serial() --- nf_core/configs/create/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/configs/create/create.py b/nf_core/configs/create/create.py index e8bc32eaf0..486c798513 100644 --- a/nf_core/configs/create/create.py +++ b/nf_core/configs/create/create.py @@ -357,7 +357,7 @@ def write_to_file(self): else: raise ValueError(f'Invalid config type: {self.config_type}') - serial_data = NextflowSerial.dumps(self.template_config.serial_hpc()) + serial_data = NextflowSerial.dumps(self.template_config.serial()) with open(filename, "w+") as file: ## Write params From 98d78b96483a81b82d59d3ef18db6d9d2bf0da6d Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 16:05:34 +1100 Subject: [PATCH 13/15] feat(serial): add withName and withLabel to config --- nf_core/configs/create/utils.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index 58849de6e2..4eb3830a32 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -169,15 +169,29 @@ def serial_pipeline(self): } for named in self.named_process_resources.keys(): - ret[named] = self.named_process_resources[named] - + ret["process"][f"withName: '{named}'"] = { + "cpus": self.named_process_resources[named]["custom_process_ncpus"], + "memory": self.named_process_resources[named]["custom_process_memgb"], + "time": self.named_process_resources[named]["custom_process_hours"] + 'h' + } + if "custom_process_queue" in self.named_process_resources[named].keys(): + ret["process"][f"withName: '{named}'"]["queue"] = self.named_process_resources[named]["custom_process_queue"] + if "executor" in self.named_process_resources[named].keys(): + ret["process"][f"withName: '{named}'"]["executor"] = self.named_process_resources[named]["custom_process_queue"] for labelled in self.labelled_process_resources.keys(): - ret[labelled] = self.labelled_process_resources[labelled] - + ret["process"][f"withLabel: '{named}'"] = { + "cpus": self.labelled_process_resources[labelled]["custom_process_ncpus"], + "memory": self.labelled_process_resources[labelled]["custom_process_memgb"], + "time": self.labelled_process_resources[labelled]["custom_process_hours"] + 'h' + } + if "custom_process_queue" in self.labelled_process_resources[labelled].keys(): + ret["process"][f"withLabel: '{named}'"]["queue"] = self.labelled_process_resources[labelled]["custom_process_queue"] + if "executor" in self.labelled_process_resources[labelled].keys(): + ret["process"][f"withLabel: '{named}'"]["executor"] = self.labelled_process_resources[labelled]["executor"] return ret def serial(self): - if self.general_config_type != "pipeline": + if self.is_infrastructure: return self.serial_hpc() else: return self.serial_pipeline() From 7bb3ad048334e77e862ec9e98c84b1f23d7ae215 Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 16:07:53 +1100 Subject: [PATCH 14/15] fix(serial): change labelled --- nf_core/configs/create/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index 4eb3830a32..73f71cf48e 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -185,9 +185,9 @@ def serial_pipeline(self): "time": self.labelled_process_resources[labelled]["custom_process_hours"] + 'h' } if "custom_process_queue" in self.labelled_process_resources[labelled].keys(): - ret["process"][f"withLabel: '{named}'"]["queue"] = self.labelled_process_resources[labelled]["custom_process_queue"] + ret["process"][f"withLabel: '{labelled}'"]["queue"] = self.labelled_process_resources[labelled]["custom_process_queue"] if "executor" in self.labelled_process_resources[labelled].keys(): - ret["process"][f"withLabel: '{named}'"]["executor"] = self.labelled_process_resources[labelled]["executor"] + ret["process"][f"withLabel: '{labelled}'"]["executor"] = self.labelled_process_resources[labelled]["executor"] return ret def serial(self): From a602113f8507acd91aaa1a4a533ff2b9528e40d6 Mon Sep 17 00:00:00 2001 From: nbtm-sh Date: Fri, 13 Mar 2026 17:50:16 +1100 Subject: [PATCH 15/15] fix(config): fix error when creating withLabel entries --- nf_core/configs/create/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/configs/create/utils.py b/nf_core/configs/create/utils.py index 73f71cf48e..8d7ad9d61e 100644 --- a/nf_core/configs/create/utils.py +++ b/nf_core/configs/create/utils.py @@ -163,8 +163,8 @@ def serial_pipeline(self): }, "process": { "cpus": self.default_process_ncpus, - "memory": self.default_process_memgb, - "time": self.default_process_hours + "memory": self.default_process_memgb + "GB", + "time": self.default_process_hours + "h" } } @@ -179,7 +179,7 @@ def serial_pipeline(self): if "executor" in self.named_process_resources[named].keys(): ret["process"][f"withName: '{named}'"]["executor"] = self.named_process_resources[named]["custom_process_queue"] for labelled in self.labelled_process_resources.keys(): - ret["process"][f"withLabel: '{named}'"] = { + ret["process"][f"withLabel: '{labelled}'"] = { "cpus": self.labelled_process_resources[labelled]["custom_process_ncpus"], "memory": self.labelled_process_resources[labelled]["custom_process_memgb"], "time": self.labelled_process_resources[labelled]["custom_process_hours"] + 'h'