Cli update (#34)

* Update extra file error to a warning and attempt to fix the problem

* Update stdout for user

* Update check-recipe
- Improve conda list pkg searching
- Add method to check for header in genomic files
- Add .csi support

* Update check-recipe test
- New tests for check-header function
- Update helper function for header tests
- New test for add_files to add extra files to the yaml
- New test for check-files to account for extra files
- test for .csi files

* Update utils with new methods
- Method to check that conda pkg dir for installed ggd metadata. If missing, will replace

* Update utils tests
- New test for checking conda pkg dir for ggd installed metadata
- Update conda version test for the newest version of conda 4.8.2

* Bump ggd version to 0.1.2

* Update install test to use better conda info searching

* Update and add new requirements
- Newest version of conda = 4.8.2
- Newest version of conda-buld = 3.8.12
- Add pysam
- Add cyvcf2

* Update check-recipe test
- Support for header check
- Support for extra file handling

* Update post link file exists handling

* Update post link file exists handling

* Update Error Hanlding

Author: mikecormier

ggd/__init__.py

@@ -1 +1,2 @@
-__version__ = "0.1.1"
+__version__ = "0.1.2"
+

ggd/check_recipe.py

@@ -27,7 +27,7 @@
 
 
 def add_check_recipe(p):
-    """Argument method used to add check-recipes as a module arugment/function """
+    """Argument method used to add check-recipe as a module arugment/function """
     import argparse
 
     c = p.add_parser(
@@ -159,7 +159,7 @@ def _build(path, recipe, debug=False):
                     [d], ggd_jdict
                 )  ## .uninstall method to remove extra ggd files
         print(
-            "\n:ggd:check-recipe: Review the STDOUT and STDERR, correct the errors, and re-run $ggd check-recipes\n"
+            "\n:ggd:check-recipe: Review the STDOUT and STDERR, correct the errors, and re-run $ggd check-recipe\n"
         )
         ## Exit
         sys.exit(5)
@@ -203,16 +203,13 @@ def _install(bz2, recipe_name, debug=False):
     3) If the installation fails progam exits. ggd data handeling is initated to remove any new/updated files from the installation process
     """
     import traceback
-    from .utils import get_required_conda_version
+    from .utils import get_conda_package_list, get_required_conda_version
 
     conda_version = get_required_conda_version()
     conda_install = "conda=" + conda_version
 
     ## See if it is already installed
-    pkg_out = sp.check_output(["conda list {}".format(recipe_name)], shell=True).decode(
-        "utf8"
-    )
-    if recipe_name in pkg_out:  ## If already installed
+    if recipe_name in get_conda_package_list(conda_root(),include_local=True).keys():
         return False
 
     ## Set CONDA_SOURCE_PREFIX environment variable
@@ -302,7 +299,7 @@ def _install(bz2, recipe_name, debug=False):
             print(e)
 
         print(
-            "\n:ggd:check-recipe: Review the STDOUT and STDERR, correct the errors, and re-run $ggd check-recipes\n"
+            "\n:ggd:check-recipe: Review the STDOUT and STDERR, correct the errors, and re-run $ggd check-recipe\n"
         )
         ## Exit
         sys.exit(1)
@@ -437,7 +434,20 @@ def check_recipe(parser, args):
 
     ## Check if previous package is already installed or it is a new installation
     if new_installed:
-        check_files(
+
+        ## Check that the file has a header
+        if not check_header(install_path):
+            print("\n:ggd:check-recipe: !!ERROR!!")
+            print(
+                "\n\t!!!!!!!!!!!!!!!!!!!!!!!\n\t! FAILED recipe check !\n\t!!!!!!!!!!!!!!!!!!!!!!!\n"
+            )
+            print(
+                "\n\t!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n\t! Recipe NOT ready for Pull Requests !\n\t!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+            )
+            remove_package_after_install(bz2, recipe_name, 1)
+            
+        ## Get the extra files
+        extra = check_files(
             install_path,
             species,
             build,
@@ -449,7 +459,7 @@ def check_recipe(parser, args):
 
         ## Add final files and md5sum
         if args.dont_add_md5sum_for_checksum == False:
-            recipe = add_final_files(install_path, recipe, args.recipe_path)
+            recipe = add_final_files(install_path, recipe, args.recipe_path, extra)
             add_to_checksum_md5sums(
                 install_path, recipe, op.join(args.recipe_path, "checksums_file.txt")
             )
@@ -598,7 +608,7 @@ def add_to_checksum_md5sums(installed_dir_path, yaml_file, recipe_checksum_file_
     return True
 
 
-def add_final_files(installed_dir_path, yaml_dict, recipe_path):
+def add_final_files(installed_dir_path, yaml_dict, recipe_path, extra_files):
     """Method to add the final data files to the meta.yaml file of a recipe
 
     add_final_files
@@ -618,6 +628,7 @@ def add_final_files(installed_dir_path, yaml_dict, recipe_path):
     1) installed_dir_path: The directory path to the installed data files
     2) yaml_dict: A dictionary of the meta.yaml file for the recipe
     3) recipe_path: The directory path to the recipe being checked 
+    4) extra_files: The name of the extra files found from check_files method
 
     Returns: 
     ++++++++
@@ -659,6 +670,7 @@ def add_final_files(installed_dir_path, yaml_dict, recipe_path):
             "mzml",
             "cvs",
             "tsv",
+            "txt",
             "bim",
             "fam",
             "ped",
@@ -689,6 +701,11 @@ def add_final_files(installed_dir_path, yaml_dict, recipe_path):
         yaml_dict["about"]["tags"]["final-files"]
     )
 
+    ## Add extra files if they exists
+    if extra_files:
+        print(":ggd:check-recipe: Attempting to add the extra files not already added in the mat.yaml file\n")
+        yaml_dict["extra"]["extra-files"] = extra_files
+
     ## Rewrite yaml file with new tags and new final files
     with open(os.path.join(recipe_path, "meta.yaml"), "w") as newFile:
         for key in sorted(yaml_dict.keys()):
@@ -804,12 +821,173 @@ def remove_package_after_install(bz2, recipe_name, exit_num):
         print(e)
 
     print(
-        "\n:ggd:check-recipe: Review the STDOUT and STDERR, correct the errors, and re-run $ggd check-recipes\n"
+        "\n:ggd:check-recipe: Review the STDOUT and STDERR, correct the errors, and re-run $ggd check-recipe\n"
     )
     ## Exit
     sys.exit(exit_num)
 
 
+def check_header(install_path):
+    """Method to check the final genomic headers have a header or not
+
+    check_header
+    ============
+    This method is going to go through each of the files that were created by the recipe, 
+     and it will check if the those files have a header or not. 
+    
+    sam/bam/cram, vcf/bcf, gtf/gff/gff3, bed/bedGraph, csv, txt files require a header and if no header is provided 
+     check-recipe will fail. 
+   
+    Other files that don't have header will be given a warning. GGD expects most files to have 
+      a header. Some files are okay not to have headers, but if a header can be added it should be. 
+
+    For each file, the file header and first 5 lines of the file body will be provided to stdout.
+
+    Parameters:
+    -----------
+    1) install_path: The path to the directory where the files have been installed into.
+
+    Returns:
+    +++++++
+    True or False. 
+     - True if a header exist or if only a warning was given
+     - False if a header does not exists and is required
+
+    """
+
+    print(":ggd:check-recipe: Checking that the final files have headers if appropriate\n")
+
+    installed_files = os.listdir(install_path)
+
+    for file_name in [x for x in installed_files if os.path.isfile(os.path.join(install_path,x))]:
+
+        f_path = os.path.join(install_path, file_name)
+
+        ## Check for an index file
+        if file_name.strip().split(".")[-1] in set(["tbi","bai","crai","fai","tar","bz2","bw"]):
+            continue 
+
+        ## Skip fasta or fastq files
+        if any(x in file_name for x in [".fasta",".fa",".fastq",".fq"]):
+            continue
+
+        ## Check for sam/bam/cram files
+        if any(x in file_name for x in [".sam",".bam",".cram"]):
+            import pysam
+
+            try:
+                samfile = pysam.AlignmentFile(f_path, check_sq=False)
+                header = samfile.header
+                if any(header.lengths):
+                    print(":ggd:check-recipe: Header found in file {name}\n".format(name=file_name))
+                    print("Head of file:")
+                    print("---------------------------")
+                    print(str(header).strip())
+                    for i,read in enumerate(samfile):
+                        print(read)
+                        if i >= 4: 
+                            break
+                    print("---------------------------\n")
+
+                else:
+                    print(":ggd:check-recipe: !!ERROR!! No header found for file {name}\n".format(name=file_name))
+                    print(":ggd:check-recipe: !!ERROR!! A header is required for sam/bam/cram files\n")
+                    return(False)
+
+            except (ValueError, IOError, Exception) as e:
+                print(str(e))
+                print(":ggd:check-recipe: !!ERROR!! No header found for file {name}\n".format(name=file_name))
+                print(":ggd:check-recipe: !!ERROR!! A header is required for sam/bam/cram files\n")
+                return(False)
+
+    
+        ## Check vcf/bcf files
+        elif any(x in file_name for x in [".vcf",".bcf"]): 
+            from cyvcf2 import VCF
+            try:
+                vcffile = VCF(f_path)
+                header = str(vcffile.raw_header)
+
+                if header:
+                    print(":ggd:check-recipe: Header found in file {name}\n".format(name=file_name))
+                    print("Head of file:")
+                    print("---------------------------")
+                    print(str(header).strip())
+                    for i,var in enumerate(vcffile):
+                        print(var)
+                        if i >= 4: 
+                            break
+                    print("---------------------------\n")
+
+                else:
+                    print(":ggd:check-recipe: !!ERROR!! No header found for file {name}\n".format(name=file_name))
+                    print(":ggd:check-recipe: !!ERROR!! A header is required for vcf/bcf files\n")
+                    return(False)
+
+            except IOError as e:
+                print(str(e))
+                print(":ggd:check-recipe: !!ERROR!! No header found for file {name}\n".format(name=file_name))
+                print(":ggd:check-recipe: !!ERROR!! A header is required for vcf/bcf files\n")
+                return(False)
+
+        ## Check other files
+        else:
+            import gzip
+            try:
+                file_handler = gzip.open(f_path) if f_path.endswith(".gz") else open(f_path)
+                header = []
+                body = []
+                try:
+                    for line in file_handler:
+
+                        if type(line) != str:
+                            line = line.strip().decode("utf-8")
+
+                        if len(line) > 0 and str(line)[0] == "#":
+                            header.append(str(line))
+
+                        else:
+                            body.append(str(line))
+                            if len(body) > 4:
+                                break
+
+                except UnicodeDecodeError:
+                    print(":ggd:check-recipe: Cannot decode file contents into unicode.\n") 
+                    pass
+
+
+                if header:
+                    print(":ggd:check-recipe: Header found in file {name}\n".format(name=file_name))
+                    print("Head of file:")
+                    print("---------------------------")
+                    print("\n".join(header))
+                    print("\n".join(body))
+                    print("---------------------------\n")
+                elif any(x in file_name for x in [".gtf", ".gff", ".gff3", ".bed", ".bedGraph", ".csv", ".txt"]):
+                    print(":ggd:check-recipe: !!ERROR!! No header found for file {name}\n".format(name=file_name))
+                    print(":ggd:check-recipe: !!ERROR!! A header is required for this type of file\n")
+                    print("First 5 lines of file body:")
+                    print("---------------------------")
+                    print("\n".join(body))
+                    print("---------------------------\n")
+                    return(False)
+                else:
+                    print(":ggd:check-recipe: !!WARNING!! No header found for file {name}\n".format(name=file_name))
+                    print("First 5 lines of file body:")
+                    print("---------------------------")
+                    print("\n".join(body))
+                    print("---------------------------\n")
+                    print(":ggd:check-recipe: !!WARNING!! GGD requires that any file that can have a header should. Please either add a header or if the file cannot have a header move forward.\n")
+                    print(":ggd:check-recipe: !!WARNING!! IF you move forwared without adding a header when one should be added, this recipe will be rejected until a header is added.\n")
+                    
+            except IOError as e:
+                print(":ggd:check-recipe: !!ERROR!!")
+                print(str(e))
+                return False
+
+    return(True)
+
+
 def check_files(
     install_path, species, build, recipe_name, extra_files, before_files, bz2
 ):
@@ -827,8 +1005,9 @@ def check_files(
     print(":ggd:check-recipe: modified files:\n\t :: %s\n\n" % "\n\t :: ".join(files))
 
     tbis = [x for x in files if x.endswith(".tbi")]  # all tbi files
+    tbis = [x for x in files if x.endswith((".tbi",".csi"))]  # all tbi files
 
-    nons = [x for x in files if not x.endswith(".tbi")]  # all non tbi files
+    nons = [x for x in files if not x.endswith((".tbi",".csi"))]  # all non tbi files
 
     tbxs = [x[:-4] for x in tbis if x[:-4] in nons]  # names of files tabixed
 
@@ -868,6 +1047,8 @@ def check_files(
     missing = []
     not_tabixed = []
     not_faidxed = []
+    add_extra = False
+    add_extra_files = []
     for n in nons:
         print(":ggd:check-recipe: > checking %s" % n)
         if n.endswith(
@@ -903,16 +1084,21 @@ def check_files(
         elif op.basename(n) not in extra_files and not any(
             fnmatch(op.basename(n), e) for e in extra_files
         ):
-            missing.append(
-                ":ggd:check-recipe: !!ERROR!!: %s(%s) unknown file and not in the extra/extra-files section of the yaml\n"
-                % (P, n)
-            )
+                print(
+                    "\n:ggd:check-recipe: !!WARNING!!: %s(%s) unknown file and not in the extra/extra-files section of the yaml\n"
+                    % (P, n) 
+                )
+                add_extra_files.append(op.basename(n))           
+                add_extra = True
 
     if missing or not_tabixed or not_faidxed:
         print("\n".join(missing + not_tabixed + not_faidxed), file=sys.stderr)
         remove_package_after_install(bz2, recipe_name, 2)
 
-    return True
+    if add_extra:
+        return add_extra_files
+    else:
+        return []
 
 
 def check_yaml(recipe):

ggd/make_bash.py

@@ -163,9 +163,7 @@ def make_bash(parser, args):
     assert (
         args.summary.strip() != ""
     ), "Please provide a thorough summary of the data package"
-    print(name)
-    print(
-        "{0}-{1}-{2}-v{3}".format(
+    print("\n:ggd:make-recipe: Name of recipe: {0}-{1}-{2}-v{3}\n".format(
             args.genome_build.lower(),
             args.name.lower(),
             data_provider.lower(),