Merge branch 'main' into develop

Contains critical pushes that are needed in develop

Author: rxu17

bin/input_to_database.py

@@ -84,8 +84,8 @@
 ```
 """
 import argparse
-from datetime import date
 import logging
+from datetime import date
 
 from genie import (
     config,
@@ -183,9 +183,9 @@ def main(
         today = date.today()
         table_name = f"Narrow MAF Database - {today}"
         # filetype = "vcf2maf"
-        # syn7208886 is the GENIE staging project to archive maf table
+        # save maf table to testing or production project as the mode
         new_tables = process_functions.create_new_fileformat_table(
-            syn, "vcf2maf", table_name, project_id, "syn7208886"
+            syn, "vcf2maf", table_name, project_id, project_id
         )
         syn.setPermissions(new_tables["newdb_ent"].id, 3326313, [])
         genie_config["vcf2maf"] = new_tables["newdb_ent"].id

genie/consortium_to_public.py

@@ -5,6 +5,7 @@
 import sys
 
 import pandas as pd
+import synapseclient
 import synapseutils
 from genie import (
     create_case_lists,
@@ -18,6 +19,9 @@
 stdout_handler = logging.StreamHandler(stream=sys.stdout)
 stdout_handler.setLevel(logging.INFO)
 logger.addHandler(stdout_handler)
+from typing import Dict
+
+from genie.load import _copyRecursive
 
 
 # TODO: Add to transform.py
@@ -338,7 +342,7 @@ def consortiumToPublic(
             )
         else:
             ent = syn.get(entId, followLink=True, downloadFile=False)
-            copiedId = synapseutils.copy(
+            copiedId = _copyRecursive(
                 syn,
                 ent,
                 public_release_preview,

genie/load.py

@@ -5,8 +5,8 @@
 
 import logging
 import os
-import time
 import tempfile
+import time
 from typing import Dict, List, Optional, Union
 
 import pandas as pd
@@ -17,6 +17,9 @@
 
 logger = logging.getLogger(__name__)
 
+import synapseutils as synu
+from synapseclient import Entity, File, Folder, Link, Project, Schema
+
 
 # TODO Edit docstring
 def store_file(
@@ -248,3 +251,128 @@ def _update_table(
         syn.store(synapseclient.Table(database_synid, update_all_file.name))
     # Delete the update file
     os.unlink(update_all_file.name)
+
+
+def _copyRecursive(
+    syn: synapseclient.Synapse,
+    entity: str,
+    destinationId: str,
+    mapping: Dict[str, str] = None,
+    skipCopyAnnotations: bool = False,
+    **kwargs,
+) -> Dict[str, str]:
+    """
+    NOTE: This is a copy of the function found here: https://github.com/Sage-Bionetworks/synapsePythonClient/blob/develop/synapseutils/copy_functions.py#L409
+    This was copied because there is a restriction that doesn't allow for copying entities with access requirements
+
+    Recursively copies synapse entites, but does not copy the wikis
+
+    Arguments:
+        syn: A Synapse object with user's login
+        entity: A synapse entity ID
+        destinationId: Synapse ID of a folder/project that the copied entity is being copied to
+        mapping: A mapping of the old entities to the new entities
+        skipCopyAnnotations: Skips copying the annotations
+                                Default is False
+
+    Returns:
+        a mapping between the original and copied entity: {'syn1234':'syn33455'}
+    """
+
+    version = kwargs.get("version", None)
+    setProvenance = kwargs.get("setProvenance", "traceback")
+    excludeTypes = kwargs.get("excludeTypes", [])
+    updateExisting = kwargs.get("updateExisting", False)
+    if mapping is None:
+        mapping = dict()
+    # Check that passed in excludeTypes is file, table, and link
+    if not isinstance(excludeTypes, list):
+        raise ValueError("Excluded types must be a list")
+    elif not all([i in ["file", "link", "table"] for i in excludeTypes]):
+        raise ValueError(
+            "Excluded types can only be a list of these values: file, table, and link"
+        )
+
+    ent = syn.get(entity, downloadFile=False)
+    if ent.id == destinationId:
+        raise ValueError("destinationId cannot be the same as entity id")
+
+    if (isinstance(ent, Project) or isinstance(ent, Folder)) and version is not None:
+        raise ValueError("Cannot specify version when copying a project of folder")
+
+    if not isinstance(ent, (Project, Folder, File, Link, Schema, Entity)):
+        raise ValueError("Not able to copy this type of file")
+
+    permissions = syn.restGET("/entity/{}/permissions".format(ent.id))
+    # Don't copy entities without DOWNLOAD permissions
+    if not permissions["canDownload"]:
+        syn.logger.warning(
+            "%s not copied - this file lacks download permission" % ent.id
+        )
+        return mapping
+
+    # HACK: These lines of code were removed to allow for data with access requirements to be copied
+    # https://github.com/Sage-Bionetworks/synapsePythonClient/blob/2909fa778e814f62f6fe6ce2d951ce58c0080a4e/synapseutils/copy_functions.py#L464-L470
+
+    copiedId = None
+
+    if isinstance(ent, Project):
+        project = syn.get(destinationId)
+        if not isinstance(project, Project):
+            raise ValueError(
+                "You must give a destinationId of a new project to copy projects"
+            )
+        copiedId = destinationId
+        # Projects include Docker repos, and Docker repos cannot be copied
+        # with the Synapse rest API. Entity views currently also aren't
+        # supported
+        entities = syn.getChildren(
+            entity, includeTypes=["folder", "file", "table", "link"]
+        )
+        for i in entities:
+            mapping = _copyRecursive(
+                syn,
+                i["id"],
+                destinationId,
+                mapping=mapping,
+                skipCopyAnnotations=skipCopyAnnotations,
+                **kwargs,
+            )
+
+        if not skipCopyAnnotations:
+            project.annotations = ent.annotations
+            syn.store(project)
+    elif isinstance(ent, Folder):
+        copiedId = synu.copy_functions._copyFolder(
+            syn,
+            ent.id,
+            destinationId,
+            mapping=mapping,
+            skipCopyAnnotations=skipCopyAnnotations,
+            **kwargs,
+        )
+    elif isinstance(ent, File) and "file" not in excludeTypes:
+        copiedId = synu.copy_functions._copyFile(
+            syn,
+            ent.id,
+            destinationId,
+            version=version,
+            updateExisting=updateExisting,
+            setProvenance=setProvenance,
+            skipCopyAnnotations=skipCopyAnnotations,
+        )
+    elif isinstance(ent, Link) and "link" not in excludeTypes:
+        copiedId = synu.copy_functions._copyLink(
+            syn, ent.id, destinationId, updateExisting=updateExisting
+        )
+    elif isinstance(ent, Schema) and "table" not in excludeTypes:
+        copiedId = synu.copy_functions._copyTable(
+            syn, ent.id, destinationId, updateExisting=updateExisting
+        )
+    # This is currently done because copyLink returns None sometimes
+    if copiedId is not None:
+        mapping[ent.id] = copiedId
+        syn.logger.info("Copied %s to %s" % (ent.id, copiedId))
+    else:
+        syn.logger.info("%s not copied" % ent.id)
+    return mapping

genie/process_mutation.py

@@ -197,6 +197,9 @@ def process_mutation_workflow(
         Annotated Maf Path. None if there are no valid mutation files.
 
     """
+    # setting maf table upload timeout time
+    syn.table_query_timeout = 50000
+
     # Get valid files
     mutation_files = validfiles["fileType"].isin(["maf", "vcf"])
     valid_mutation_files = validfiles["path"][mutation_files].tolist()