Merge pull request #8229 from 4teamwork/es/TI-3678-fix-tus-upload

Avoid loading file content into RAM during TUS upload deserialization

Author: elioschmutz

changes/TI-3678.bugfix

@@ -0,0 +1 @@
+Fix huge memory counsumption if uploading a large file [elioschmutz]

opengever/api/configure.zcml

@@ -79,6 +79,7 @@
   <adapter factory=".field_deserializers.DateFieldDeserializer" />
   <adapter factory=".field_deserializers.DatetimeFieldDeserializer" />
   <adapter factory=".field_deserializers.UTCDatetimeFieldDeserializer" />
+  <adapter factory=".field_deserializers.GeverNamedFieldDeserializer" />
   <adapter factory=".todo.SerializeToDoToJson" />
 
   <configure zcml:condition="installed z3c.relationfield">

opengever/api/field_deserializers.py

@@ -5,16 +5,20 @@
 from opengever.base.schema import IUTCDatetime
 from persistent.interfaces import IPersistent
 from plone.dexterity.interfaces import IDexterityContent
+from plone.namedfile.interfaces import INamedField
 from plone.restapi.deserializer.dxfields import DatetimeFieldDeserializer
 from plone.restapi.deserializer.dxfields import DefaultFieldDeserializer
+from plone.restapi.deserializer.dxfields import NamedFieldDeserializer
 from plone.restapi.interfaces import IFieldDeserializer
+from plone.restapi.services.content.tus import TUSUpload
 from pytz import utc
 from zope.component import adapter
 from zope.interface import implementer
 from zope.publisher.interfaces.browser import IBrowserRequest
 from zope.schema.interfaces import IDate
 from zope.schema.interfaces import IDatetime
 from zope.schema.interfaces import IField
+from zope.schema.interfaces import RequiredMissing
 import dateutil
 
 
@@ -84,3 +88,35 @@ def reject_year_before_1900(value):
     if isinstance(value, (date, datetime)) and value.year < 1900:
         raise ValueError(
             'year=%s is invalid. Year must be >= 1900.' % value.year)
+
+
+class TUSUploadNamedField(object):
+    """Lightweight named file field used during TUS uploads instead of the
+    original INamedField, to avoid loading the entire file content into RAM
+    during field level validations.
+    """
+
+    def __init__(self, original_field):
+        self.required = original_field.required
+        self.missing_value = original_field.missing_value
+        self.__name__ = original_field.__name__
+        self._type = original_field._type
+
+    def validate(self, value):
+        if value == self.missing_value and self.required:
+            raise RequiredMissing(self.__name__)
+
+
+@implementer(IFieldDeserializer)
+@adapter(INamedField, IDexterityContent, IOpengeverBaseLayer)
+class GeverNamedFieldDeserializer(NamedFieldDeserializer):
+    """For TUS uploads, replaces self.field with a TUSUploadNamedField before
+    delegating to the parent. The parent's self.field.validate(value) call then
+    hits TUSUploadNamedField.validate(), which only checks the required
+    constraint without loading the blob content into RAM.
+    """
+
+    def __call__(self, value):
+        if isinstance(value, TUSUpload):
+            self.field = TUSUploadNamedField(self.field)
+        return super(GeverNamedFieldDeserializer, self).__call__(value)