Fix pickling and add docs

Author: ddelange

docs/backends/amazon-S3.rst

@@ -259,13 +259,25 @@ Settings
     Setting this overrides the settings for ``addressing_style``, ``signature_version`` and
     ``proxies``. Include them as arguments to your ``botocore.config.Config`` class if you need them.
 
+``client_ttl`` or ``AWS_S3_CLIENT_TTL``
+
+  Default: ``3600``
+
+  The amount of seconds to cache a boto3 client resource.
+
+  .. note::
+
+    Long-lived boto3 clients have a known `memory leak`_, which is why the client needs to be
+    periodically recreated to avoid excessive memory consumption.
+
 .. _AWS Signature Version 4: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
 .. _S3 region list: https://docs.aws.amazon.com/general/latest/gr/s3.html#s3_region
 .. _list of canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 .. _Boto3 docs for uploading files: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.put_object
 .. _Boto3 docs for TransferConfig: https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.TransferConfig
 .. _ManifestStaticFilesStorage: https://docs.djangoproject.com/en/3.1/ref/contrib/staticfiles/#manifeststaticfilesstorage
 .. _Botocore docs: https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html#botocore.config.Config
+.. _memory leak: https://github.com/boto/boto3/issues/1670
 
 .. _cloudfront-signed-url-header:
 

storages/backends/s3.py

@@ -7,6 +7,7 @@
 import warnings
 from datetime import datetime
 from datetime import timedelta
+from functools import cached_property
 from urllib.parse import urlencode
 
 from django.contrib.staticfiles.storage import ManifestFilesMixin
@@ -379,10 +380,6 @@ def __init__(self, **settings):
             else:
                 self.cloudfront_signer = None
 
-        self._ttl_cache = cachetools.cached(
-            cache=cachetools.TTLCache(maxsize=2, ttl=3600), lock=threading.Lock()
-        )
-
     def get_cloudfront_signer(self, key_id, key):
         cache_key = f"{key_id}:{key}"
         if cache_key not in self.__class__._signers:
@@ -451,6 +448,7 @@ def get_default_settings(self):
             "use_threads": setting("AWS_S3_USE_THREADS", True),
             "transfer_config": setting("AWS_S3_TRANSFER_CONFIG", None),
             "client_config": setting("AWS_S3_CLIENT_CONFIG", None),
+            "client_ttl": setting("AWS_S3_CLIENT_TTL", 3600),
         }
 
     def __getstate__(self):
@@ -465,23 +463,27 @@ def __setstate__(self, state):
     def connection(self):
         """
         Get the (cached) thread-safe boto3 s3 resource.
-
-        This function has a 1 hour time to live cache for the boto3 resource.
-        We want to avoid storing a resource for too long to avoid their memory leak
-        ref https://github.com/boto/boto3/issues/1670.
         """
         return self._ttl_cache(self._create_connection)()
 
     @property
     def unsigned_connection(self):
         """
         Get the (cached) thread-safe boto3 s3 resource (unsigned).
+        """
+        return self._ttl_cache(self._create_connection)(unsigned=True)
 
-        This function has a 1 hour time to live cache for the boto3 resource.
+    @cached_property
+    def _ttl_cache(self):
+        """
+        This time to live cache is used to periodically recreate boto3 clients.
         We want to avoid storing a resource for too long to avoid their memory leak
         ref https://github.com/boto/boto3/issues/1670.
         """
-        return self._ttl_cache(self._create_connection)(unsigned=True)
+        return cachetools.cached(
+            cache=cachetools.TTLCache(maxsize=2, ttl=self.client_ttl),
+            lock=threading.Lock(),
+        )
 
     def _create_connection(self, *, unsigned=False):
         """

tests/test_s3.py

@@ -75,7 +75,9 @@ def test_pickle(self):
         _ = storage.connection
         _ = storage.bucket
         p = pickle.dumps(storage)
-        pickle.loads(p)
+        new_storage = pickle.loads(p)
+        _ = new_storage.connection
+        _ = storage.bucket
 
     def test_storage_url_slashes(self):
         """