Added serology, host_epigenetics_raw_seq_set and host_variant_call nodes, tests, and examples.

Author: Victor

cutlass/HostEpigeneticsRawSeqSet.py

@@ -5,7 +5,9 @@
 from .Cytokine import Cytokine
 from .HostSeqPrep import HostSeqPrep
 from .HostAssayPrep import HostAssayPrep
+from .HostEpigeneticsRawSeqSet import HostEpigeneticsRawSeqSet
 from .HostTranscriptomicsRawSeqSet import HostTranscriptomicsRawSeqSet
+from .HostVariantCall import HostVariantCall
 from .HostWgsRawSeqSet import HostWgsRawSeqSet
 from .Lipidome import Lipidome
 from .Metabolome import Metabolome
@@ -16,6 +18,7 @@
 from .ProteomeNonPride import ProteomeNonPride
 from .Sample import Sample
 from .SampleAttribute import SampleAttribute
+from .Serology import Serology
 from .SixteenSDnaPrep import SixteenSDnaPrep
 from .SixteenSRawSeqSet import SixteenSRawSeqSet
 from .SixteenSTrimmedSeqSet import SixteenSTrimmedSeqSet

cutlass/HostVariantCall.py

@@ -1,12 +1,12 @@
 #!/usr/bin/python
 
+""" Wrapper module for ascp usage. """
+
 import os
 import re
 import subprocess
 import logging
 
-# Test wrapper code for ascp
-
 # download example command(s):
 #
 # only getting ~ 10Mb/s (on 20-30Mb connection): with defaults:
@@ -29,35 +29,47 @@
 
 # compare version numbers
 def version_cmp(v1, v2):
+    """
+    Compare version/release numbers.
+    """
     logger.debug("In version_cmp.")
     def normalize(v):
-        return [int(x) for x in re.sub(r'(\.0+)*$','', v).split(".")]
+        """ Normalize a dotted version string. """
+        return [int(x) for x in re.sub(r'(\.0+)*$', '', v).split(".")]
     return cmp(normalize(v1), normalize(v2))
 
-# Return version number of ascp executable referenced by ascp_command.
-# May raise an exception if the path is invalid.
-def get_ascp_version(ascp_command):
+def get_ascp_version():
+    """
+    Return version number of ascp executable referenced by ascp_command.
+    May raise an exception if the path is invalid.
+    """
     logger.debug("In get_ascp_version.")
 
     version = None
-    output = subprocess.check_output([ascp_command, "--version"],
-                                     universal_newlines = True)
+    output = subprocess.check_output([ASCP_COMMAND, "--version"],
+                                     universal_newlines=True)
+
     cre = re.compile(r"^.+version (\d[\d\.]+)", re.MULTILINE)
     for match in cre.finditer(output):
         version = match.groups()[0]
 
     if version is None:
-        raise Exception("Output from ascp command ('" + ascp_command + \
+        raise Exception("Output from ascp command ('" + ASCP_COMMAND + \
                         " --version') did not contain a recognizable " + \
                         "version number.")
     return version
 
-def check_ascp_version(ascp_command="ascp"):
+def check_ascp_version():
+    """
+    Check that the ascp utility is installed and that its version
+    is within an acceptable range. If the utility is not present,
+    or the version is unacceptable, an exception is raised.
+    """
     logger.debug("In check_ascp_version.")
 
     # check ascp version, raise error if too low
     try:
-        ascp_ver = get_ascp_version(ascp_command)
+        ascp_ver = get_ascp_version()
     except:
         raise Exception("Unable to determine ascp version. Is it installed?")
 
@@ -67,40 +79,49 @@ def check_ascp_version(ascp_command="ascp"):
     return True
 
 def get_ascp_env(password):
+    """
+    Get the environment dictionary after adding the ASPERA_SCP_PASS variable
+    (and value) to it.
+    """
     logger.debug("In get_ascp_env.")
 
-    e = os.environ.copy()
-    if 'ASPERA_SCP_PASS' in e:
+    environment = os.environ.copy()
+    if 'ASPERA_SCP_PASS' in environment:
         logger.info("Honoring previously set ASPERA_SCP_PASS environment variable.")
     else:
         if password != None:
             logger.info("Setting ASPERA_SCP_PASS environment variable.")
-            e['ASPERA_SCP_PASS'] = password
+            environment['ASPERA_SCP_PASS'] = password
 
-    return e
+    return environment
 
-# run ascp command, returning True for success or False for failure
 def run_ascp(ascp_cmd, password, keyfile=None):
+    """
+    Run the ascp command, returning True for success or False for failure.
+    """
     logger.debug("In run_ascp.")
 
     if keyfile:
         if not os.path.exists(keyfile):
             raise IOError(
-                "Can't use private key. No such file or directory: "+keyfile)
+                "Can't use private key. No such file or directory: " + keyfile)
         ascp_cmd = [ascp_cmd[0], "-i", keyfile] + ascp_cmd[1:]
 
     try:
-        logger.debug("Command: " + " ".join(ascp_cmd))
-        process = subprocess.Popen(ascp_cmd, stdout=subprocess.PIPE,
-                             stdin=subprocess.PIPE,
-                             stderr=subprocess.PIPE,
-                             universal_newlines = True,
-                             env = get_ascp_env(password))
+        logger.debug("Command: %s", " ".join(ascp_cmd))
+        process = subprocess.Popen(
+            ascp_cmd,
+            stdout=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            env=get_ascp_env(password)
+        )
 
         logger.info("Beginning transfer.")
         (s_out, s_err) = process.communicate()
         rc = process.returncode
-        logger.info("Invocation of ascp complete. Return code: %s." % str(rc))
+        logger.info("Invocation of ascp complete. Return code: %s.", str(rc))
 
         success = False
 
@@ -112,36 +133,46 @@ def run_ascp(ascp_cmd, password, keyfile=None):
                 logger.error("Aspera authentication failure.")
             else:
                 if s_err != None:
-                    logger.error("Unexpected STDERR from ascp: %s" % s_err)
+                    logger.error("Unexpected STDERR from ascp: %s", s_err)
                 if s_out != None:
-                    logger.error("Unexpected STDOUT from ascp: %s" % s_out)
+                    logger.error("Unexpected STDOUT from ascp: %s", s_out)
     except subprocess.CalledProcessError as cpe:
-        logger.error("Encountered an error when running ascp: ", cpe)
+        logger.error("Encountered an error when running ascp: %s", cpe)
 
     return success
 
-# download a single file via Aspera. return True if successful, False if not
 def download_file(server, username, password, remote_path, local_path,
                   keyfile=None):
+    """
+    Download a single remote file using the aspera ascp utility.
+    Returns True if successful, False if not.
+    """
     logger.debug("In download_file.")
 
-    check_ascp_version(ASCP_COMMAND)
-    ascp_cmd = [ ASCP_COMMAND, "-T", "-v", "-l", "300M", username + "@" +
-                 server + ":" + remote_path, local_path ]
+    check_ascp_version()
+    ascp_cmd = [
+        ASCP_COMMAND, "-T", "-v", "-l", "300M",
+        username + "@" + server + ":" + remote_path,
+        local_path
+    ]
+
     return run_ascp(ascp_cmd, password, keyfile)
 
-# upload a single file via Aspera. return True if successful, False if not
 def upload_file(server, username, password, local_file, remote_path,
                 keyfile=None):
+    """
+    Upload a single file with the Aspera ascp utility.
+    Return True if successful, False if not.
+    """
     logger.debug("In upload_file.")
-    check_ascp_version(ASCP_COMMAND)
+    check_ascp_version()
 
     # check that local file exists
     if not os.path.isfile(local_file):
         logger.warn("local file " + local_file + " does not exist")
         return False
 
     remote_clause = username + "@" + server + ":" + remote_path
-    ascp_cmd = [ ASCP_COMMAND, "-T", "-v", "-l", "300M", local_file, remote_clause ]
+    ascp_cmd = [ASCP_COMMAND, "-T", "-v", "-l", "300M", local_file, remote_clause]
 
     return run_ascp(ascp_cmd, password, keyfile)

cutlass/Serology.py

@@ -67,8 +67,10 @@ def _get_cutlass_instance(self, name):
             "clustered_seq_set"                  : "ClusteredSeqSet",
             "cytokine"                           : "Cytokine",
             "host_assay_prep"                    : "HostAssayPrep",
+            "host_epigenetics_raw_seq_set"       : "HostEpigeneticsRawSeqSet",
             "host_seq_prep"                      : "HostSeqPrep",
             "host_transcriptomics_raw_seq_set"   : "HostTranscriptomicsRawSeqSet",
+            "host_variant_call"                  : "HostVariantCall",
             "host_wgs_raw_seq_set"               : "HostWgsRawSeqSet",
             "lipidome"                           : "Lipidome",
             "metabolome"                         : "Metabolome",
@@ -79,6 +81,7 @@ def _get_cutlass_instance(self, name):
             "proteome_nonpride"                  : "ProteomeNonPride",
             "sample"                             : "Sample",
             "sample_attr"                        : "SampleAttribute",
+            "serology"                           : "Serology",
             "study"                              : "Study",
             "subject"                            : "Subject",
             "subject_attr"                       : "SubjectAttribute",
@@ -210,7 +213,7 @@ def port(self):
     @enforce_int
     def port(self, port):
         """
-        The port setter
+        The port setter.
 
         Args:
             port (int): The new port for OSDF access.
@@ -237,7 +240,7 @@ def server(self):
     @enforce_string
     def server(self, server):
         """
-        The server setter
+        The server setter.
 
         Args:
             server (str): The new server for OSDF access.
@@ -264,7 +267,7 @@ def ssl(self):
     @enforce_bool
     def ssl(self, ssl):
         """
-        The ssl setter
+        The ssl setter.
 
         Args:
             ssl (bool): Whether to use encryption (SSL) or not.
@@ -291,7 +294,7 @@ def username(self):
     @enforce_string
     def username(self, username):
         """
-        The username setter
+        The username setter.
 
         Args:
             username (str): The new username for OSDF access.

cutlass/__init__.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+# pylint: disable=C0111, C0325
+
+import logging
+import sys
+import tempfile
+from pprint import pprint
+from cutlass import HostEpigeneticsRawSeqSet
+from cutlass import iHMPSession
+
+username = "test"
+password = "test"
+
+def set_logging():
+    """ Setup logging. """
+    root = logging.getLogger()
+    root.setLevel(logging.DEBUG)
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    root.addHandler(ch)
+
+set_logging()
+
+session = iHMPSession(username, password)
+
+print("Required fields: ")
+print(HostEpigeneticsRawSeqSet.required_fields())
+
+seq_set = HostEpigeneticsRawSeqSet()
+
+seq_set.checksums = {"md5": "72bdc024d83226ccc90fbd2177e78d56"}
+seq_set.assay_type = "RRBS"
+seq_set.comment = "test comment. Hello world!"
+seq_set.exp_length = 2000
+seq_set.format = "fasta"
+seq_set.format_doc = "url"
+seq_set.seq_model = "a machine"
+seq_set.sequence_type = "nucleotide"
+seq_set.size = 3000
+seq_set.study = "prediabetes"
+
+print("Creating a temp file for example/testing purposes.")
+temp_file = tempfile.NamedTemporaryFile(delete=False).name
+print("Local file: %s" % temp_file)
+
+seq_set.local_file = temp_file
+
+# Optional properties
+seq_set.private_files = False
+
+seq_set.links = {"sequenced_from": ["88af6472fb03642dd5eaf8cddc70c8ec"]}
+
+seq_set.tags = ["host_epigenetics_raw_seq_set", "ihmp"]
+seq_set.add_tag("another")
+seq_set.add_tag("and_another")
+
+print(seq_set.to_json(indent=2))
+
+if seq_set.is_valid():
+    print("Valid!")
+
+    success = seq_set.save()
+
+    if success:
+        seq_set_id = seq_set.id
+        print("Successfully saved prep. ID: %s" % seq_set_id)
+
+        seq_set2 = HostEpigeneticsRawSeqSet.load(seq_set_id)
+
+        print(seq_set.to_json(indent=4))
+
+        deletion_success = seq_set.delete()
+
+        if deletion_success:
+            print("Deleted sequence set with ID %s" % seq_set_id)
+        else:
+            print("Deletion of sequence set %s failed." % seq_set_id)
+    else:
+        print("Save failed")
+else:
+    print("Invalid...")
+    validation_errors = seq_set.validate()
+    pprint(validation_errors)