v1.3.4 update (#122)

Author: dthoward96

README.Rmd

@@ -26,7 +26,7 @@ github_pages_url <- description$GITHUB_PAGES
 
 <p style="font-size: 16px;"><em>Public Database Submission Pipeline</em></p>
 
-**Beta Version**: v1.3.3. This pipeline is currently in Beta testing, and issues could appear during submission. Please use it at your own risk. Feedback and suggestions are welcome! 
+**Beta Version**: v1.3.4. This pipeline is currently in Beta testing, and issues could appear during submission. Please use it at your own risk. Feedback and suggestions are welcome! 
 
 **General Disclaimer**: This repository was created for use by CDC programs to collaborate on public health related projects in support of the [CDC mission](https://www.cdc.gov/about/organization/mission.htm).  GitHub is not hosted by the CDC, but is a third party website used by CDC and its partners to share information and collaborate on software. CDC use of GitHub does not imply an endorsement of any one particular service, product, or enterprise.
 

README.md

@@ -9,7 +9,7 @@
 
 <!-- ![GitHub last commit](https://img.shields.io/github/last-commit/montilab/cadra) -->
 
-**Beta Version**: 1.3.3. This pipeline is currently in Beta testing, and
+**Beta Version**: 1.3.4. This pipeline is currently in Beta testing, and
 issues could appear during submission. Please use it at your own risk.
 Feedback and suggestions are welcome\!
 

argument_handler.py

@@ -23,7 +23,7 @@ def args_parser():
 	config_file_parser = argparse.ArgumentParser(add_help=False)
 	file_parser = argparse.ArgumentParser(add_help=False)
 	test_parser = argparse.ArgumentParser(add_help=False)
-
+	publication_parser = argparse.ArgumentParser(add_help=False)
 	database_parser.add_argument("--biosample", "-b",
 		help="Create/Submit BioSample data.",
 		action="store_const",
@@ -87,6 +87,15 @@ def args_parser():
 		action="store_const",
 		default=False,
 		const=True)
+	publication_parser.add_argument("--publication_title",
+		help="Publication Title associated with sample submission. For GenBank only, overwrites value given via config file.",
+		required=False,
+		default=None)
+	publication_parser.add_argument("--publication_status",
+		help="Status of publication associated with sample submission. For GenBank only, overwrites value given via config file.",
+		required=False,
+		default=None,
+		choices=["Unpublished", "In-press", "Published"])
 
 	# Create the submodule commands
 	subparser_modules = parser.add_subparsers(dest="command")
@@ -96,15 +105,15 @@ def args_parser():
 		"prep",
 		formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 		description="Generate all files required to submit to databases selected.",
-		parents=[database_parser, organism_parser, submission_name_parser, submission_dir_parser, config_file_parser, file_parser, validate_parser]
+		parents=[database_parser, organism_parser, submission_name_parser, submission_dir_parser, config_file_parser, file_parser, validate_parser, publication_parser]
 	)
 
 	# submit command
 	submit_module = subparser_modules.add_parser(
 		"submit",
 		formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 		description="Generate all files required and begin the submission process to databases selected.",
-		parents=[database_parser, organism_parser, submission_name_parser, submission_dir_parser, config_file_parser, file_parser, test_parser, validate_parser]
+		parents=[database_parser, organism_parser, submission_name_parser, submission_dir_parser, config_file_parser, file_parser, test_parser, validate_parser, publication_parser]
 	)
 
 	# check_submission_status command

biosample_sra_handler.py

@@ -22,7 +22,7 @@ def check_raw_read_files(submission_name: str, submission_dir: str, metadata: pd
 	raw_reads_path_default = os.path.join(os.path.split(os.path.split(submission_dir)[0])[0], "raw_reads")
 	# Separate samples stored in local and cloud
 	local_df = metadata[metadata["sra-file_location"] == "local"]
-	file_columns = [col for col in local_df.columns if re.match("sra-file_[1-9]\d*", col)]
+	file_columns = [col for col in local_df.columns if re.match(r"sra-file_[1-9]\d*", col)]
 	validated_files = set()
 	invalid_raw_files = []
 	for index, row in local_df.iterrows():
@@ -58,7 +58,7 @@ def create_manual_submission_files(database: str, submission_dir: str, metadata:
 		column_ordered = ["sample_name","library_ID"]
 		prefix = "sra-"
 		# Create SRA specific fields
-		filename_cols = [col for col in metadata.columns.tolist() if re.match("sra-file_[1-9]\d*", col)]
+		filename_cols = [col for col in metadata.columns.tolist() if re.match(r"sra-file_[1-9]\d*", col)]
 		# Correct index for filename column
 		for col in filename_cols:
 			# Remove 0 index
@@ -176,7 +176,7 @@ def create_submission_xml(organism: str, database: str, submission_name: str, co
 	if "SRA" in database:
 		database_df = metadata.filter(regex=(SRA_REGEX)).copy()
 		database_df = database_df.drop_duplicates()
-		file_columns = [col for col in database_df.columns if re.match("sra-file_[1-9]\d*", col)]
+		file_columns = [col for col in database_df.columns if re.match(r"sra-file_[1-9]\d*", col)]
 		for index, row in database_df.iterrows():
 			action = etree.SubElement(root, "Action")
 			addfiles = etree.SubElement(action, "AddFiles", target_db="SRA")
@@ -197,7 +197,7 @@ def create_submission_xml(organism: str, database: str, submission_name: str, co
 				datatype = etree.SubElement(file, "DataType")
 				datatype.text = "generic-data"
 			# Remove columns with sra- prefix that are not attributes
-			sra_cols = [col for col in database_df.columns.tolist() if col.startswith('sra-') and not re.match("(sra-sample_name|sra-title|sra-comment|sra-file_location|sra-file_\d*)", col)]
+			sra_cols = [col for col in database_df.columns.tolist() if col.startswith('sra-') and not re.match(r"(sra-sample_name|sra-title|sra-comment|sra-file_location|sra-file_\d*)", col)]
 			for col in sra_cols:
 				attribute_value = row[col]
 				if pd.notnull(attribute_value) and attribute_value.strip() != "":

config/seqsender/config_file/gisaid_schema.py

@@ -203,6 +203,11 @@
                         'type': 'integer',
                         'allowed': [1, 2],
                         'nullable': True
+                    },
+                    'CLI_Path': {
+                        'required': False,
+                        'type': 'string',
+                        'nullable': True
                     }
                 }
             }

config/seqsender/config_file/ncbi_gisaid_schema.py

@@ -185,6 +185,11 @@
                         'type': ['integer', 'string'],
                         'allowed': [1, 2, 'None'],
                         'nullable': True
+                    },
+                    'CLI_Path': {
+                        'required': False,
+                        'type': 'string',
+                        'nullable': True
                     }
                 }
             }

config/seqsender/config_file/ncbi_schema.py

@@ -188,6 +188,11 @@
                         'type': 'integer',
                         'allowed': [1, 2],
                         'nullable': True
+                    },
+                    'CLI_Path': {
+                        'required': False,
+                        'type': 'string',
+                        'nullable': True
                     }
                 }
             }

config/seqsender/upload_log_schema.py

@@ -17,7 +17,7 @@
 		"Organism": Column(
 			dtype="object",
 			checks=[
-				Check.isin(["FLU", "COV", "POX", "ARBO", "OTHER"]),
+				Check.isin(["FLU", "COV", "POX", "ARBO", "RSV", "OTHER"]),
 			],
 			nullable=False,
 			unique=False,

docs/app.json

@@ -6,7 +6,7 @@
 
 import sys
 import os
-from typing import Dict
+from typing import Dict, Any
 import pandas as pd
 from Bio import SeqIO
 from Bio.Seq import Seq
@@ -33,7 +33,7 @@ def validate_directory(name: str, path: str):
 		sys.exit(1)
 
 # Validate gisaid cli exists or error out
-def validate_gisaid_installer(submission_dir: str, organism: str) -> str:
+def validate_gisaid_installer(submission_dir: str, organism: str, config_dict: Dict[str, Any]) -> str:
 	# /<submission_dir>/gisaid_cli/<organism>_CLI
 	gisaid_cli_path_option_one = os.path.join(submission_dir, "gisaid_cli", organism.lower()+"CLI")
 	# /seqsender/gisaid_cli/<organism>_CLI
@@ -42,7 +42,10 @@ def validate_gisaid_installer(submission_dir: str, organism: str) -> str:
 	gisaid_cli_path_option_three = os.path.join(submission_dir, "gisaid_cli", organism.lower()+"CLI", organism.lower()+"CLI")
 	# /seqsender>/gisaid_cli/<organism>_CLI/<organism>_CLI
 	gisaid_cli_path_option_four = os.path.join(PROG_DIR, "gisaid_cli", organism.lower()+"CLI", organism.lower()+"CLI")
-	if os.path.isfile(gisaid_cli_path_option_one):
+	# gisaid cli path provided by config file
+	if "CLI_Path" in config_dict and config_dict["CLI_Path"] is not None and config_dict["CLI_Path"].strip() != "" and os.path.isfile(config_dict["CLI_Path"].strip()):
+		return config_dict["CLI_Path"].strip()
+	elif os.path.isfile(gisaid_cli_path_option_one):
 		return gisaid_cli_path_option_one
 	elif os.path.isfile(gisaid_cli_path_option_two):
 		return gisaid_cli_path_option_two
@@ -51,6 +54,9 @@ def validate_gisaid_installer(submission_dir: str, organism: str) -> str:
 	elif os.path.isfile(gisaid_cli_path_option_four):
 		return gisaid_cli_path_option_four
 	else:
+		if "CLI_Path" in config_dict and config_dict["CLI_Path"] is not None and config_dict["CLI_Path"].strip() != "":
+			cli_path_error = config_dict["CLI_Path"]
+			print(f"Error: There is not a GISAID CLI for {organism} provided via config file at: '{cli_path_error}'", file=sys.stderr)
 		print(f"Error: There is not a GISAID CLI for {organism} located at: '{gisaid_cli_path_option_one}' or '{gisaid_cli_path_option_two}'", file=sys.stderr)
 		print(f"Download the GISAID CLI for {organism} from \"https://gisaid.org/\".", file=sys.stderr)
 		print(f"Extract the zip file and place the CLI binary at either: '{gisaid_cli_path_option_one}' or '{gisaid_cli_path_option_two}'", file=sys.stderr)