services: metadata input on zenodo deposit creation

* adds serializers/validation for metadata input
* adds unit tests for zenodo serializer
* closes #1952

Signed-off-by: Ilias Koutsakis <[email protected]>

Author: Lilykos

cap/modules/deposit/api.py

@@ -61,14 +61,15 @@
 from cap.modules.repos.tasks import download_repo, download_repo_file
 from cap.modules.repos.utils import (create_webhook, disconnect_subscriber,
                                      parse_git_url)
+from cap.modules.services.serializers.zenodo import ZenodoUploadSchema
 from cap.modules.schemas.resolvers import (resolve_schema_by_url,
                                            schema_name_to_url)
 from cap.modules.user.errors import DoesNotExistInLDAP
 from cap.modules.user.utils import (get_existing_or_register_role,
                                     get_existing_or_register_user)
 
 from .errors import (DepositValidationError, UpdateDepositPermissionsError,
-                     ReviewError)
+                     ReviewError, InputValidationError)
 from .fetchers import cap_deposit_fetcher
 from .minters import cap_deposit_minter
 from .permissions import (AdminDepositPermission, CloneDepositPermission,
@@ -269,12 +270,22 @@ def upload(self, pid, *args, **kwargs):
                             'Please connect your Zenodo account '
                             'before creating a deposit.')
 
-                    files = data.get('files')
+                    files = data.get('files', [])
                     bucket = data.get('bucket')
-                    zenodo_data = data.get('zenodo_data', {})
+                    zenodo_data = data.get('zenodo_data')
+
+                    input = {'files': files, 'bucket': bucket}
+                    if zenodo_data:
+                        input['data'] = zenodo_data
 
                     if files and bucket:
-                        zenodo_deposit = create_zenodo_deposit(token, zenodo_data)  # noqa
+                        payload, errors = ZenodoUploadSchema().load(input)
+                        if errors:
+                            raise InputValidationError(
+                                'Validation error in Zenodo input data.',
+                                errors=errors)
+
+                        zenodo_deposit = create_zenodo_deposit(token, payload)
                         self.setdefault('_zenodo', []).append(zenodo_deposit)
                         self.commit()
 

cap/modules/deposit/errors.py

@@ -138,6 +138,21 @@ def __init__(self, description, errors=None, **kwargs):
         self.errors = [FieldError(e[0], e[1]) for e in errors.items()]
 
 
+class InputValidationError(RESTValidationError):
+    """Review validation error exception."""
+
+    code = 400
+
+    description = "Validation error. Try again with valid data"
+
+    def __init__(self, description, errors=None, **kwargs):
+        """Initialize exception."""
+        super(InputValidationError, self).__init__(**kwargs)
+
+        self.description = description or self.description
+        self.errors = [FieldError(e[0], e[1]) for e in errors.items()]
+
+
 class DataValidationError(RESTValidationError):
     """Review validation error exception."""
 

cap/modules/deposit/tasks.py

@@ -28,7 +28,6 @@
 import requests
 from flask import current_app
 from celery import shared_task
-from invenio_db import db
 from invenio_files_rest.models import FileInstance, ObjectVersion
 
 

cap/modules/deposit/utils.py

@@ -25,15 +25,16 @@
 
 from __future__ import absolute_import, print_function
 
+import json
 import requests
 from flask import current_app
-from flask_login import current_user
 from invenio_access.models import Role
 from invenio_db import db
 
 from cap.modules.deposit.errors import AuthorizationError, \
     DataValidationError, FileUploadError
 from cap.modules.records.utils import url_to_api_url
+from cap.modules.services.serializers.zenodo import ZenodoDepositSchema
 
 
 def clean_empty_values(data):
@@ -82,13 +83,16 @@ def add_api_to_links(links):
     return response
 
 
-def create_zenodo_deposit(token, data):
+def create_zenodo_deposit(token, data=None):
     """Create a Zenodo deposit using the logged in user's credentials."""
     zenodo_url = current_app.config.get("ZENODO_SERVER_URL")
+    zenodo_data = data.get('data')
+    upload_data = {'metadata': zenodo_data} if zenodo_data else {}
+
     deposit = requests.post(
         url=f'{zenodo_url}/deposit/depositions',
         params=dict(access_token=token),
-        json={'metadata': data},
+        json=upload_data,
         headers={'Content-Type': 'application/json'}
     )
 
@@ -105,18 +109,5 @@ def create_zenodo_deposit(token, data):
         raise FileUploadError(
             'Something went wrong, Zenodo deposit not created.')
 
-    # TODO: fix with serializers
-    data = deposit.json()
-    zenodo_deposit = {
-        'id': data['id'],
-        'title': data.get('metadata', {}).get('title'),
-        'creator': current_user.id,
-        'created': data['created'],
-        'links': {
-            'self': data['links']['self'],
-            'bucket': data['links']['bucket'],
-            'html': data['links']['html'],
-            'publish': data['links']['publish'],
-        }
-    }
-    return zenodo_deposit
+    data = ZenodoDepositSchema().dump(deposit.json()).data
+    return data

cap/modules/services/serializers/zenodo.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of CERN Analysis Preservation Framework.
+# Copyright (C) 2020 CERN.
+#
+# CERN Analysis Preservation Framework is free software; you can redistribute
+# it and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# CERN Analysis Preservation Framework is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with CERN Analysis Preservation Framework; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+# MA 02111-1307, USA.
+#
+# In applying this license, CERN does not
+# waive the privileges and immunities granted to it by virtue of its status
+# as an Intergovernmental Organization or submit itself to any jurisdiction.
+# or submit itself to any jurisdiction.
+
+"""Zenodo Serializer/Validator."""
+
+import arrow
+from flask_login import current_user
+from marshmallow import Schema, fields, ValidationError, validate, validates, \
+    validates_schema
+
+from invenio_files_rest.models import ObjectVersion
+
+DATE_REGEX = r'\d{4}-\d{2}-\d{2}'
+DATE_ERROR = 'The date should follow the pattern YYYY-mm-dd.'
+CHOICE_ERROR = lambda choices: f'Not a valid choice. Select one of: {choices}'  # noqa
+
+UPLOAD_TYPES = [
+    'publication',
+    'poster',
+    'presentation',
+    'dataset',
+    'image',
+    'video',
+    'software',
+    'lesson',
+    'physicalobject',
+    'other'
+]
+LICENSES = [
+    'CC-BY-4.0',
+    'CC-BY-1.0',
+    'CC-BY-2.0',
+    'CC-BY-3.0'
+]
+ACCESS_RIGHTS = [
+    'open',
+    'embargoed',
+    'restricted',
+    'closed'
+]
+
+
+class ZenodoCreatorsSchema(Schema):
+    name = fields.String(required=True)
+    affiliation = fields.String()
+    orcid = fields.String()
+
+
+class ZenodoDepositMetadataSchema(Schema):
+    title = fields.String(required=True)
+    description = fields.String(required=True)
+    version = fields.String()
+
+    keywords = fields.List(fields.String())
+    creators = fields.List(
+        fields.Nested(ZenodoCreatorsSchema), required=True)
+
+    upload_type = fields.String(required=True, validate=validate.OneOf(
+            UPLOAD_TYPES, error=CHOICE_ERROR(UPLOAD_TYPES)))
+    license = fields.String(required=True, validate=validate.OneOf(
+            LICENSES, error=CHOICE_ERROR(LICENSES)))
+    access_right = fields.String(required=True, validate=validate.OneOf(
+        ACCESS_RIGHTS, error=CHOICE_ERROR(ACCESS_RIGHTS)))
+
+    publication_date = fields.String(
+        required=True, validate=validate.Regexp(DATE_REGEX, error=DATE_ERROR))
+    embargo_date = fields.String(
+        validate=validate.Regexp(DATE_REGEX, error=DATE_ERROR))
+    access_conditions = fields.String()
+
+    @validates('embargo_date')
+    def validate_embargo_date(self, value):
+        """Validate that embargo date is in the future."""
+        if arrow.get(value).date() <= arrow.utcnow().date():
+            raise ValidationError(
+                'Embargo date must be in the future.',
+                field_names=['embargo_date']
+            )
+
+    @validates_schema()
+    def validate_license(self, data, **kwargs):
+        """Validate license."""
+        access = data.get('access_right')
+        if access in ['open', 'embargoed'] and 'license' not in data:
+            raise ValidationError(
+                'Required when access right is open or embargoed.',
+                field_names=['license']
+            )
+        if access == 'embargoed' and 'embargo_date' not in data:
+            raise ValidationError(
+                'Required when access right is embargoed.',
+                field_names=['embargo_date']
+            )
+        if access == 'restricted' and 'access_conditions' not in data:
+            raise ValidationError(
+                'Required when access right is restricted.',
+                field_names=['access_conditions']
+            )
+
+
+class ZenodoUploadSchema(Schema):
+    files = fields.List(fields.String(), required=True)
+    data = fields.Nested(ZenodoDepositMetadataSchema, default=dict())
+    bucket = fields.String(required=True)
+
+    @validates_schema()
+    def validate_files(self, data, **kwargs):
+        bucket = data['bucket']
+        files = data['files']
+
+        for _file in files:
+            obj = ObjectVersion.get(bucket, _file)
+            if not obj:
+                raise ValidationError(
+                    f'File {_file} not found in bucket.',
+                    field_names=['files']
+                )
+
+
+class ZenodoDepositSchema(Schema):
+    id = fields.Int(dump_only=True)
+    created = fields.String(dump_only=True)
+
+    title = fields.Method('get_title', dump_only=True, allow_none=True)
+    creator = fields.Method('get_creator', dump_only=True, allow_none=True)
+    links = fields.Method('get_links', dump_only=True)
+
+    def get_creator(self, data):
+        return current_user.id if current_user else None
+
+    def get_title(self, data):
+        return data.get('metadata', {}).get('title')
+
+    def get_links(self, data):
+        return {
+            'self': data['links']['self'],
+            'bucket': data['links']['bucket'],
+            'html': data['links']['html'],
+            'publish': data['links']['publish']
+        }