[SYNPY-1437] INCOMPLETE/DRAFT - Start the process to move to use the /bundle2 rest API

synapseclient/api/__init__.py

@@ -20,6 +20,7 @@
     get_entity_id_version_bundle2,
     post_entity_bundle2_create,
     put_entity_id_bundle2,
+    store_entity_with_bundle2,
 )
 from .entity_factory import get_from_entity_factory
 from .entity_services import (
@@ -64,6 +65,7 @@
     "get_entity_id_version_bundle2",
     "post_entity_bundle2_create",
     "put_entity_id_bundle2",
+    "store_entity_with_bundle2",
     # file_services
     "post_file_multipart",
     "put_file_multipart_add",

synapseclient/api/entity_bundle_services_v2.py

@@ -152,3 +152,92 @@ async def put_entity_id_bundle2(
         + (f"?generatedBy={generated_by}" if generated_by else ""),
         body=json.dumps(request),
     )
+
+
+async def store_entity_with_bundle2(
+    entity: Dict[str, Any],
+    parent_id: Optional[str] = None,
+    acl: Optional[Dict[str, Any]] = None,  # TODO: Consider skipping ACL?
+    annotations: Optional[Dict[str, Any]] = None,
+    activity: Optional[Dict[str, Any]] = None,
+    new_version: bool = False,
+    force_version: bool = False,
+    *,
+    synapse_client: Optional["Synapse"] = None,
+) -> Dict[str, Any]:
+    """
+    Store an entity in Synapse using the bundle2 API endpoints to reduce HTTP calls.
+
+    This function follows a specific flow:
+    1. Determines if the operation is a create or update:
+       - If no ID is provided, searches for the ID via /entity/child
+       - If no ID is found, treats as a Create
+       - If an ID is found, treats as an Update
+
+    2. For Updates:
+       - Retrieves entity by ID and merges with existing data
+       - Updates desired fields in the retrieved object
+       - Pushes modified object with HTTP PUT if there are changes
+
+    3. For Creates:
+       - Creates a new object with desired fields
+       - Pushes the new object with HTTP POST
+
+    Arguments:
+        entity: The entity to store.
+        parent_id: The ID of the parent entity for creation.
+        acl: Access control list for the entity.
+        annotations: Annotations to associate with the entity.
+        activity: Activity to associate with the entity.
+        new_version: If True, create a new version of the entity.
+        force_version: If True, forces a new version of an entity even if nothing has changed.
+        synapse_client: Synapse client instance.
+
+    Returns:
+        The stored entity bundle.
+    """
+    from synapseclient import Synapse
+
+    client = Synapse.get_client(synapse_client=synapse_client)
+
+    # Determine if this is a create or update operation
+    entity_id = entity.get("id", None)
+
+    # Construct bundle request based on provided data
+    bundle_request = {"entity": entity}
+
+    if annotations:
+        bundle_request["annotations"] = annotations
+
+    if acl:
+        bundle_request["accessControlList"] = acl
+
+    if activity:
+        bundle_request["activity"] = activity
+
+    # Handle create or update
+    if not entity_id:
+        # This is a creation
+        client.logger.debug("Creating new entity via bundle2 API")
+
+        # For creation, parent ID is required
+        # TODO: Projects won't have a parent in this case
+        # if parent_id:
+        #     # Add parentId to the entity if not already set
+        #     if not entity.get("parentId"):
+        #         entity["parentId"] = parent_id
+        # elif not entity.get("parentId"):
+        #     raise ValueError("Parent ID must be provided for entity creation")
+
+        # Create entity using bundle2 create endpoint
+        return await post_entity_bundle2_create(
+            request=bundle_request,
+            generated_by=activity.get("id") if activity else None,
+            synapse_client=synapse_client,
+        )
+    else:
+        # This is an update
+        client.logger.debug(f"Updating entity {entity_id} via bundle2 API")
+
+        # For updates we might need to retrieve the existing entity to merge data
+        # Only retrieve if we need

synapseclient/api/entity_factory.py

@@ -245,7 +245,7 @@ async def _handle_file_entity(
     from synapseclient.models import FileHandle
 
     entity_instance.fill_from_dict(
-        synapse_file=entity_bundle["entity"], set_annotations=False
+        synapse_file=entity_bundle["entity"], annotations=None
     )
 
     # Update entity with FileHandle metadata
@@ -401,7 +401,7 @@ class type. This will also download the file if `download_file` is set to True.
         )
     else:
         # Handle all other entity types
-        entity_instance.fill_from_dict(entity_bundle["entity"], set_annotations=False)
+        entity_instance.fill_from_dict(entity_bundle["entity"], annotations=None)
 
     if annotations:
         entity_instance.annotations = annotations

synapseclient/models/annotations.py

@@ -1,13 +1,16 @@
 """The required data for working with annotations in Synapse"""
 
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from datetime import date, datetime
 from typing import Dict, List, Optional, Union
 
+from typing_extensions import Any
+
 from synapseclient import Synapse
-from synapseclient.annotations import ANNO_TYPE_TO_FUNC
+from synapseclient.annotations import ANNO_TYPE_TO_FUNC, _convert_to_annotations_list
 from synapseclient.api import set_annotations_async
 from synapseclient.core.async_utils import async_to_sync
+from synapseclient.core.utils import delete_none_keys
 from synapseclient.models.protocols.annotations_protocol import (
     AnnotationsSynchronousProtocol,
 )
@@ -136,3 +139,23 @@ def from_dict(
                 annotations[key] = dict_to_convert[key]
 
         return annotations
+
+    def to_synapse_request(self) -> Dict[str, Any]:
+        """Convert the annotations to the format the synapse rest API works in.
+
+        Returns:
+            The annotations in the format the synapse rest API works in.
+        """
+        annotations_dict = asdict(self)
+
+        synapse_annotations = _convert_to_annotations_list(
+            annotations_dict["annotations"] or {}
+        )
+
+        result = {
+            "annotations": synapse_annotations,
+            "id": self.id,
+            "etag": self.etag,
+        }
+        delete_none_keys(result)
+        return result

synapseclient/models/dataset.py

@@ -855,12 +855,12 @@ def _set_last_persistent_instance(self) -> None:
             [dataclasses.replace(item) for item in self.items] if self.items else []
         )
 
-    def fill_from_dict(self, entity, set_annotations: bool = True) -> "Self":
+    def fill_from_dict(self, entity, annotations: Dict = None) -> "Self":
         """
         Converts the data coming from the Synapse API into this datamodel.
 
         Arguments:
-            synapse_table: The data coming from the Synapse API
+            entity: The data coming from the Synapse API
 
         Returns:
             The Dataset object instance.
@@ -887,8 +887,8 @@ def fill_from_dict(self, entity, set_annotations: bool = True) -> "Self":
             for item in entity.get("items", [])
         ]
 
-        if set_annotations:
-            self.annotations = Annotations.from_dict(entity.get("annotations", {}))
+        if annotations:
+            self.annotations = Annotations.from_dict(annotations.get("annotations", {}))
         return self
 
     def to_synapse_request(self):
@@ -2255,12 +2255,12 @@ def _set_last_persistent_instance(self) -> None:
             [dataclasses.replace(item) for item in self.items] if self.items else []
         )
 
-    def fill_from_dict(self, entity, set_annotations: bool = True) -> "Self":
+    def fill_from_dict(self, entity, annotations: Dict = None) -> "Self":
         """
         Converts the data coming from the Synapse API into this datamodel.
 
         Arguments:
-            synapse_table: The data coming from the Synapse API
+            entity: The data coming from the Synapse API
 
         Returns:
             The DatasetCollection object instance.
@@ -2283,8 +2283,8 @@ def fill_from_dict(self, entity, set_annotations: bool = True) -> "Self":
             EntityRef(id=item["entityId"], version=item["versionNumber"])
             for item in entity.get("items", [])
         ]
-        if set_annotations:
-            self.annotations = Annotations.from_dict(entity.get("annotations", {}))
+        if annotations:
+            self.annotations = Annotations.from_dict(annotations.get("annotations", {}))
         return self
 
     def to_synapse_request(self):

synapseclient/models/entityview.py

@@ -739,9 +739,7 @@ def _set_last_persistent_instance(self) -> None:
             deepcopy(self.scope_ids) if self.scope_ids else set()
         )
 
-    def fill_from_dict(
-        self, entity: Dict, set_annotations: bool = True
-    ) -> "EntityView":
+    def fill_from_dict(self, entity: Dict, annotations: Dict = None) -> "EntityView":
         """
         Converts the data coming from the Synapse API into this datamodel.
 
@@ -768,8 +766,8 @@ def fill_from_dict(
         self.view_type_mask = entity.get("viewTypeMask", None)
         self.scope_ids = set(f"syn{id}" for id in entity.get("scopeIds", []))
 
-        if set_annotations:
-            self.annotations = Annotations.from_dict(entity.get("annotations", {}))
+        if annotations:
+            self.annotations = Annotations.from_dict(annotations.get("annotations", {}))
         return self
 
     def to_synapse_request(self):

synapseclient/models/file.py

@@ -6,7 +6,7 @@
 from copy import deepcopy
 from dataclasses import dataclass, field
 from datetime import date, datetime
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from synapseclient import File as SynapseFile
 from synapseclient import Synapse
@@ -607,7 +607,7 @@ def _fill_from_file_handle(self) -> None:
     def fill_from_dict(
         self,
         synapse_file: Union[Synapse_File, Dict[str, Union[bool, str, int]]],
-        set_annotations: bool = True,
+        annotations: Dict = None,
     ) -> "File":
         """
         Converts a response from the REST API into this dataclass.
@@ -642,10 +642,8 @@ def fill_from_dict(
             )
             self._fill_from_file_handle()
 
-        if set_annotations:
-            self.annotations = Annotations.from_dict(
-                synapse_file.get("annotations", {})
-            )
+        if annotations:
+            self.annotations = Annotations.from_dict(annotations.get("annotations", {}))
         return self
 
     def _cannot_store(self) -> bool:
@@ -866,10 +864,16 @@ async def store_async(
             delete_none_keys(synapse_file)
 
             entity = await store_entity(
-                resource=self, entity=synapse_file, synapse_client=client
+                entity=self.to_synapse_request(),
+                parent_id=self.parent_id,
+                annotations=Annotations(self.annotations).to_synapse_request()
+                if self.annotations
+                else None,
+                synapse_client=synapse_client,
+            )
+            self.fill_from_dict(
+                entity=entity["entity"], annotations=entity.get("annotations", None)
             )
-
-            self.fill_from_dict(synapse_file=entity, set_annotations=False)
 
         re_read_required = await store_entity_components(
             root_resource=self, synapse_client=client
@@ -948,7 +952,9 @@ async def change_metadata_async(
             ),
         )
 
-        self.fill_from_dict(synapse_file=entity, set_annotations=True)
+        self.fill_from_dict(
+            synapse_file=entity, annotations=entity.get("annotations", {})
+        )
         self._set_last_persistent_instance()
         Synapse.get_client(synapse_client=synapse_client).logger.debug(
             f"Change metadata for file {self.name}, id: {self.id}: {self.path}"
@@ -1391,3 +1397,32 @@ def _convert_into_legacy_file(self) -> SynapseFile:
         )
         delete_none_keys(return_data)
         return return_data
+
+    def to_synapse_request(self) -> Dict[str, Any]:
+        """
+        Converts this dataclass into a request that can be sent to the Synapse API.
+
+        Returns:
+            A dictionary that can be used in a request to the Synapse API.
+        """
+        from synapseclient.core.constants import concrete_types
+
+        entity = {
+            "name": self.name,
+            "description": self.description,
+            "id": self.id,
+            "etag": self.etag,
+            "createdOn": self.created_on,
+            "modifiedOn": self.modified_on,
+            "createdBy": self.created_by,
+            "modifiedBy": self.modified_by,
+            "parentId": self.parent_id,
+            "dataFileHandleId": self.data_file_handle_id,
+            "versionLabel": self.version_label,
+            "versionComment": self.version_comment,
+            "versionNumber": self.version_number,
+            "concreteType": concrete_types.FILE_ENTITY,
+        }
+        delete_none_keys(entity)
+
+        return entity