First beta for v4.x

pyscicat/client.py

@@ -16,12 +16,17 @@
     Attachment,
     Datablock,
     Dataset,
+    RawDataset,
     DerivedDataset,
+    UpdateDataset,
+    UpdateRawDataset,
+    UpdateDerivedDataset,
     Instrument,
     OrigDatablock,
     Proposal,
-    RawDataset,
     Sample,
+    Ownable,
+    MongoQueryable,
 )
 
 logger = logging.getLogger("splash_ingest")
@@ -82,39 +87,56 @@ def __init__(
         self._password = password  # default password
         self._token = token  # store token here
         self._headers = {}  # store headers
+        self._exclude_fields = {
+            'default': set(MongoQueryable.__fields__.keys())
+        }
 
         if not self._token:
             assert (self._username is not None) and (
                 self._password is not None
             ), "SciCat login credentials (username, password) must be provided if token is not provided"
             self._token = get_token(self._base_url, self._username, self._password)
-            self._headers["Authorization"] = "Bearer {}".format(self._token)
+        if not self._token:
+            logger.error("Token not provided")
+            raise ScicatCommError("Token not provided")
+        self._headers["Authorization"] = "Bearer {}".format(self._token)
 
-    def _send_to_scicat(self, cmd: str, endpoint: str, data: BaseModel = None):
+
+    def _send_to_scicat(
+        self, 
+        cmd: str, 
+        endpoint: str, 
+        data: BaseModel = None,
+        exclude_fields: set = {},
+    ):
         """sends a command to the SciCat API server using url and token, returns the response JSON
         Get token with the getToken method"""
         return requests.request(
             method=cmd,
             url=urljoin(self._base_url, endpoint),
-            json=data.dict(exclude_none=True) if data is not None else None,
+            json=data.dict(exclude=exclude_fields,exclude_none=True,exclude_unset=True) if data is not None else None,
             params={"access_token": self._token},
             headers=self._headers,
             timeout=self._timeout_seconds,
             stream=False,
             verify=True,
         )
 
+
     def _call_endpoint(
         self,
         cmd: str,
         endpoint: str,
         data: BaseModel = None,
         operation: str = "",
         allow_404=False,
+        exclude_fields: set = {},
     ) -> Optional[dict]:
-        response = self._send_to_scicat(cmd=cmd, endpoint=endpoint, data=data)
-        result = response.json()
+        response = self._send_to_scicat(cmd=cmd, endpoint=endpoint, data=data, exclude_fields=exclude_fields)
+        #print(response)
+        #print(response.text)
         if not response.ok:
+            result = response.json()
             err = result.get("error", {})
             if (
                 allow_404
@@ -125,6 +147,7 @@ def _call_endpoint(
                 logger.error("Error in operation %s: %s", operation, err)
                 return None
             raise ScicatCommError(f"Error in operation {operation}: {err}")
+        result = response.json()
         logger.info(
             "Operation '%s' successful%s",
             operation,
@@ -162,7 +185,7 @@ def datasets_replace(self, dataset: Dataset) -> str:
             )
         return self._call_endpoint(
             cmd="post", endpoint=dataset_url, data=dataset, operation="datasets_replace"
-        ).get("pid")
+        )
 
     """
         Upload or create a new dataset
@@ -195,8 +218,12 @@ def datasets_create(self, dataset: Dataset) -> str:
             Raises if a non-20x message is returned
         """
         return self._call_endpoint(
-            cmd="post", endpoint="Datasets", data=dataset, operation="datasets_create"
-        ).get("pid")
+            cmd="post", 
+            endpoint="Datasets", 
+            data=dataset, 
+            operation="datasets_create",
+            exclude_fields=self._exclude_fields['default'],
+        )
 
     """
         Upload a new dataset
@@ -233,7 +260,7 @@ def datasets_raw_replace(self, dataset: Dataset) -> str:
             endpoint="RawDataSets/replaceOrCreate",
             data=dataset,
             operation="datasets_raw_replace",
-        ).get("pid")
+        )
 
     """
         Upload a raw dataset
@@ -270,17 +297,17 @@ def datasets_derived_replace(self, dataset: Dataset) -> str:
             endpoint="DerivedDataSets/replaceOrCreate",
             data=dataset,
             operation="datasets_derived_replace",
-        ).get("pid")
+        )
 
-    def datasets_update(self, dataset: Dataset, pid: str) -> str:
+    def datasets_update(self, dataset: UpdateDataset, pid: str) -> str:
         """Updates an existing dataset
         This function was renamed.
         It is still accessible with the original name for backward compatibility
         The original name was update_dataset.
 
         Parameters
         ----------
-        dataset : Dataset
+        dataset : UpdateDataset
             Dataset to update
 
         pid
@@ -300,7 +327,7 @@ def datasets_update(self, dataset: Dataset, pid: str) -> str:
             endpoint=f"Datasets/{quote_plus(pid)}",
             data=dataset,
             operation="datasets_update",
-        ).get("pid")
+        )
 
     """
         Update a dataset
@@ -317,7 +344,7 @@ def datasets_datablock_create(
         It is still accessible with the original name for backward compatibility
         The original names were create_dataset_datablock and upload_datablock
         This function is obsolete and will be removed in future releases
-        Function datasets_origdatablock_create should be used.
+        Function datasets_datablock_create should be used.
 
         Parameters
         ----------
@@ -378,6 +405,7 @@ def datasets_origdatablock_create(self, origdatablock: OrigDatablock) -> dict:
             endpoint=endpoint,
             data=origdatablock,
             operation="datasets_origdatablock_create",
+            exclude_fields={'id','datasetId','ownerGroup','accessGroups'},
         )
 
     """
@@ -452,7 +480,8 @@ def samples_create(self, sample: Sample) -> str:
             endpoint="Samples",
             data=sample,
             operation="samples_create",
-        ).get("sampleId")
+            exclude_fields=self._exclude_fields['default'],
+        )
 
     upload_sample = samples_create
 
@@ -489,7 +518,7 @@ def samples_update(self, sample: Sample, sampleId: str = None) -> str:
             endpoint=f"Samples/{quote_plus(sampleId)}",
             data=sample,
             operation="samples_update",
-        ).get("sampleId")
+        )
 
     def instruments_create(self, instrument: Instrument):
         """
@@ -519,7 +548,8 @@ def instruments_create(self, instrument: Instrument):
             endpoint="Instruments",
             data=instrument,
             operation="instruments_create",
-        ).get("pid")
+            exclude_fields=self._exclude_fields['default'],
+        )
 
     upload_instrument = instruments_create
 
@@ -558,7 +588,7 @@ def instruments_update(self, instrument: Instrument, pid: str = None) -> str:
             endpoint=f"Instruments/{quote_plus(pid)}",
             data=instrument,
             operation="instruments_update",
-        ).get("pid")
+        )
 
     def proposals_create(self, proposal: Proposal):
         """
@@ -588,7 +618,7 @@ def proposals_create(self, proposal: Proposal):
             endpoint="Proposals",
             data=proposal,
             operation="proposals_create",
-        ).get("proposalId")
+        )
 
     upload_proposal = proposals_create
 
@@ -626,7 +656,7 @@ def proposals_update(self, proposal: Proposal, proposalId: str = None) -> str:
             endpoint=f"Proposals/{quote_plus(proposalId)}",
             data=proposal,
             operation="proposals_update",
-        ).get("proposalId")
+        )
 
     def datasets_find(
         self, skip: int = 0, limit: int = 25, query_fields: Optional[dict] = None
@@ -677,7 +707,14 @@ def datasets_find(
     get_datasets_full_query = datasets_find
     find_datasets_full_query = datasets_find
 
-    def datasets_get_many(self, filter_fields: Optional[dict] = None) -> Optional[dict]:
+    def datasets_get_many(
+            self, 
+            full_filter: Optional[dict] = None,
+            filter_fields: Optional[dict] = None,
+            where: Optional[dict] = None,
+            fields: Optional[list] = None,
+            limits: Optional[dict] = None
+    ) -> Optional[dict]:
         """
         Gets datasets using the simple fiter mechanism. This
         is appropriate when you do not require paging or text search, but
@@ -700,15 +737,56 @@ def datasets_get_many(self, filter_fields: Optional[dict] = None) -> Optional[di
 
         Parameters
         ----------
+        full_filter : dict
+            Dictionary with all the options included.
+            This parameter has precendence on the others, if specified, the others are ignored
+
         filter_fields : dict
             Dictionary of filtering fields. Must be json serializable.
+            This parameter is deprecated and will be removed in future releases.
+            Please use where parameter
+
+        where : dict
+            Dictionary containing the where conditions to apply in the search
+
+        fields : list
+            List of the fields to be returned in the results
+
+        limits : dict
+            List of the limits to apply in the search.
+            This parameter can contain the following fields:
+            - limit : number indicating how many results have to be returned
+            - skip : number indicating how many items needs to be skipped in the beginning of the list
+            - order : enumeration (ascending or descending) indicating the order of the results
         """
-        if not filter_fields:
-            filter_fields = {}
-        filter_fields = json.dumps(filter_fields)
-        endpoint = f'/Datasets/?filter={{"where":{filter_fields}}}'
+
+        #if not filter_fields:
+        #    filter_fields = {}
+        #filter_fields = json.dumps(filter_fields)
+
+        filter_dict = {}
+        if full_filter:
+            filter_dict = full_filter
+        else:
+            if where:
+                filter_dict['where'] = where
+            elif filter_fields:
+                filter_dict['where'] = filter_fields
+
+            if fields:
+                filter_dict['fields'] = fields
+
+            if limits:
+                filter_dict['limits'] = limits
+
+        filter_string = json.dumps(filter_dict) if filter_dict else ""
+        endpoint = 'Datasets' + f'?filter={filter_string}' if filter_string else ""
+        #print(endpoint)
         return self._call_endpoint(
-            cmd="get", endpoint=endpoint, operation="datasets_get_many", allow_404=True
+            cmd="get", 
+            endpoint=endpoint, 
+            operation="datasets_get_many", 
+            allow_404=True
         )
 
     """
@@ -879,7 +957,7 @@ def datasets_origdatablocks_get_one(self, pid: str) -> Optional[dict]:
         """
         return self._call_endpoint(
             cmd="get",
-            endpoint=f"/Datasets/{quote_plus(pid)}/origdatablocks",
+            endpoint=f"Datasets/{quote_plus(pid)}/origdatablocks",
             operation="datasets_origdatablocks_get_one",
             allow_404=True,
         )
@@ -904,13 +982,73 @@ def datasets_delete(self, pid: str) -> Optional[dict]:
         """
         return self._call_endpoint(
             cmd="delete",
-            endpoint=f"/Datasets/{quote_plus(pid)}",
+            endpoint=f"Datasets/{quote_plus(pid)}",
             operation="datasets_delete",
             allow_404=True,
         )
 
     delete_dataset = datasets_delete
 
+    def origdatablocks_create(self, origdatablock: OrigDatablock) -> dict:
+        """
+        Create a new SciCat Dataset OrigDatablock
+        This function has been renamed.
+        It is still accessible with the original name for backward compatibility
+        The original names were create_dataset_origdatablock and upload_dataset_origdatablock
+
+        Parameters
+        ----------
+        origdatablock :
+            The OrigDatablock to create
+
+        Returns
+        -------
+        dict
+            The created OrigDatablock with id
+
+        Raises
+        ------
+        ScicatCommError
+            Raises if a non-20x message is returned
+
+        """
+        endpoint = f"origdatablocks"
+        return self._call_endpoint(
+            cmd="post",
+            endpoint=endpoint,
+            data=origdatablock,
+            operation="origdatablock_create",
+            exclude_fields=self._exclude_fields['default'],
+        )
+
+    def origdatablocks_delete(self, oid: str) -> Optional[dict]:
+        """
+        Delete existing SciCat Dataset OrigDatablock
+
+        Parameters
+        ----------
+        oid :
+            The OrigDatablock id to be deleted
+
+        Returns
+        -------
+        dict
+            The OrigDatablock that has been deleted
+
+        Raises
+        ------
+        ScicatCommError
+            Raises if a non-20x message is returned
+
+        """
+        endpoint = f"origdatablocks/{oid}"
+        return self._call_endpoint(
+            cmd="delete",
+            endpoint=endpoint,
+            operation="origdatablock_delete",
+            allow_404=True,
+        )
+
 
 def get_file_size(pathobj):
     filesize = pathobj.lstat().st_size