[GEN-2094] Use customized _copyRecursive to enable to copy files with AR (#598)

* add _copyrecursive to copy files with AR

Author: danlu1

genie/consortium_to_public.py

@@ -5,6 +5,7 @@
 import sys
 
 import pandas as pd
+import synapseclient
 import synapseutils
 from genie import (
     create_case_lists,
@@ -18,6 +19,9 @@
 stdout_handler = logging.StreamHandler(stream=sys.stdout)
 stdout_handler.setLevel(logging.INFO)
 logger.addHandler(stdout_handler)
+from typing import Dict
+
+from genie.load import _copyRecursive
 
 
 # TODO: Add to transform.py
@@ -338,7 +342,7 @@ def consortiumToPublic(
             )
         else:
             ent = syn.get(entId, followLink=True, downloadFile=False)
-            copiedId = synapseutils.copy(
+            copiedId = _copyRecursive(
                 syn,
                 ent,
                 public_release_preview,

genie/load.py

@@ -5,8 +5,8 @@
 
 import logging
 import os
-import time
 import tempfile
+import time
 from typing import Dict, List, Optional, Union
 
 import pandas as pd
@@ -17,6 +17,9 @@
 
 logger = logging.getLogger(__name__)
 
+import synapseutils as synu
+from synapseclient import Entity, File, Folder, Link, Project, Schema
+
 
 # TODO Edit docstring
 def store_file(
@@ -248,3 +251,128 @@ def _update_table(
         syn.store(synapseclient.Table(database_synid, update_all_file.name))
     # Delete the update file
     os.unlink(update_all_file.name)
+
+
+def _copyRecursive(
+    syn: synapseclient.Synapse,
+    entity: str,
+    destinationId: str,
+    mapping: Dict[str, str] = None,
+    skipCopyAnnotations: bool = False,
+    **kwargs,
+) -> Dict[str, str]:
+    """
+    NOTE: This is a copy of the function found here: https://github.com/Sage-Bionetworks/synapsePythonClient/blob/develop/synapseutils/copy_functions.py#L409
+    This was copied because there is a restriction that doesn't allow for copying entities with access requirements
+
+    Recursively copies synapse entites, but does not copy the wikis
+
+    Arguments:
+        syn: A Synapse object with user's login
+        entity: A synapse entity ID
+        destinationId: Synapse ID of a folder/project that the copied entity is being copied to
+        mapping: A mapping of the old entities to the new entities
+        skipCopyAnnotations: Skips copying the annotations
+                                Default is False
+
+    Returns:
+        a mapping between the original and copied entity: {'syn1234':'syn33455'}
+    """
+
+    version = kwargs.get("version", None)
+    setProvenance = kwargs.get("setProvenance", "traceback")
+    excludeTypes = kwargs.get("excludeTypes", [])
+    updateExisting = kwargs.get("updateExisting", False)
+    if mapping is None:
+        mapping = dict()
+    # Check that passed in excludeTypes is file, table, and link
+    if not isinstance(excludeTypes, list):
+        raise ValueError("Excluded types must be a list")
+    elif not all([i in ["file", "link", "table"] for i in excludeTypes]):
+        raise ValueError(
+            "Excluded types can only be a list of these values: file, table, and link"
+        )
+
+    ent = syn.get(entity, downloadFile=False)
+    if ent.id == destinationId:
+        raise ValueError("destinationId cannot be the same as entity id")
+
+    if (isinstance(ent, Project) or isinstance(ent, Folder)) and version is not None:
+        raise ValueError("Cannot specify version when copying a project of folder")
+
+    if not isinstance(ent, (Project, Folder, File, Link, Schema, Entity)):
+        raise ValueError("Not able to copy this type of file")
+
+    permissions = syn.restGET("/entity/{}/permissions".format(ent.id))
+    # Don't copy entities without DOWNLOAD permissions
+    if not permissions["canDownload"]:
+        syn.logger.warning(
+            "%s not copied - this file lacks download permission" % ent.id
+        )
+        return mapping
+
+    # HACK: These lines of code were removed to allow for data with access requirements to be copied
+    # https://github.com/Sage-Bionetworks/synapsePythonClient/blob/2909fa778e814f62f6fe6ce2d951ce58c0080a4e/synapseutils/copy_functions.py#L464-L470
+
+    copiedId = None
+
+    if isinstance(ent, Project):
+        project = syn.get(destinationId)
+        if not isinstance(project, Project):
+            raise ValueError(
+                "You must give a destinationId of a new project to copy projects"
+            )
+        copiedId = destinationId
+        # Projects include Docker repos, and Docker repos cannot be copied
+        # with the Synapse rest API. Entity views currently also aren't
+        # supported
+        entities = syn.getChildren(
+            entity, includeTypes=["folder", "file", "table", "link"]
+        )
+        for i in entities:
+            mapping = _copyRecursive(
+                syn,
+                i["id"],
+                destinationId,
+                mapping=mapping,
+                skipCopyAnnotations=skipCopyAnnotations,
+                **kwargs,
+            )
+
+        if not skipCopyAnnotations:
+            project.annotations = ent.annotations
+            syn.store(project)
+    elif isinstance(ent, Folder):
+        copiedId = synu.copy_functions._copyFolder(
+            syn,
+            ent.id,
+            destinationId,
+            mapping=mapping,
+            skipCopyAnnotations=skipCopyAnnotations,
+            **kwargs,
+        )
+    elif isinstance(ent, File) and "file" not in excludeTypes:
+        copiedId = synu.copy_functions._copyFile(
+            syn,
+            ent.id,
+            destinationId,
+            version=version,
+            updateExisting=updateExisting,
+            setProvenance=setProvenance,
+            skipCopyAnnotations=skipCopyAnnotations,
+        )
+    elif isinstance(ent, Link) and "link" not in excludeTypes:
+        copiedId = synu.copy_functions._copyLink(
+            syn, ent.id, destinationId, updateExisting=updateExisting
+        )
+    elif isinstance(ent, Schema) and "table" not in excludeTypes:
+        copiedId = synu.copy_functions._copyTable(
+            syn, ent.id, destinationId, updateExisting=updateExisting
+        )
+    # This is currently done because copyLink returns None sometimes
+    if copiedId is not None:
+        mapping[ent.id] = copiedId
+        syn.logger.info("Copied %s to %s" % (ent.id, copiedId))
+    else:
+        syn.logger.info("%s not copied" % ent.id)
+    return mapping