Remove storage entry if resumable storage has been enabled and the stored URL has been voided (#105)

* Add new test case when stored url has been voided

* Implement special case handler when initialization raises error due to voided url in the storage

Author: pandabear

tests/test_uploader.py

@@ -1,5 +1,6 @@
 import os
 import io
+import tempfile
 from base64 import b64encode
 from unittest import mock
 
@@ -91,6 +92,51 @@ def test_url(self, filename: str):
         self.assertEqual(resumable_uploader.url, "http://tusd.tusdemo.net/files/foo_bar")
         self.assertEqual(resumable_uploader.offset, 10)
 
+    @parametrize(
+        "filename",
+        [FILEPATH_TEXT, FILEPATH_BINARY],
+    )
+    @responses.activate
+    def test_url_voided(self, filename: str):
+        # Test that voided stored url are cleared
+        responses.add(
+            responses.POST,
+            self.client.url,
+            adding_headers={"location": "http://tusd.tusdemo.net/files/foo"},
+        )
+        responses.add(
+            responses.HEAD,
+            "http://tusd.tusdemo.net/files/foo",
+            status=404,
+        )
+
+        # Create temporary storage file.
+        temp_fp = tempfile.NamedTemporaryFile(delete=False)
+        storage = filestorage.FileStorage(temp_fp.name)
+        uploader = self.client.uploader(
+            file_path=filename, store_url=True, url_storage=storage
+        )
+
+        # Conduct only POST creation so that we'd get a storage entry.
+        uploader.upload(stop_at=-1)
+        key = uploader._get_fingerprint()
+        # First ensure that an entry was created and stored.
+        self.assertIsNotNone(uploader.url)
+        self.assertIsNotNone(storage.get_item(key))
+
+        # Now start a new upload, resuming where we left off.
+        resumed_uploader = self.client.uploader(
+            file_path=filename, store_url=True, url_storage=storage
+        )
+        # HEAD response was 404 so url and storage has to be voided.
+        self.assertIsNone(resumed_uploader.url)
+        self.assertIsNone(storage.get_item(key))
+
+        # Remove the temporary storage file.
+        storage.close()
+        temp_fp.close()
+        os.remove(temp_fp.name)
+
     def test_request_length(self):
         self.uploader.chunk_size = 200
         self.assertEqual(self.uploader.get_request_length(), 200)
@@ -203,19 +249,19 @@ def test_upload_empty(self):
         # Upload URL being set means the POST request was sent and the empty
         # file was uploaded without a single PATCH request.
         self.assertTrue(uploader.url)
-    
+
     @mock.patch('tusclient.uploader.uploader.TusRequest')
     def test_upload_checksum(self, request_mock):
         self.mock_request(request_mock)
         self.uploader.upload_checksum = True
         self.uploader.upload()
         self.assertEqual(self.uploader.offset, self.uploader.get_file_size())
-    
+
     @parametrize("chunk_size", [1, 2, 3, 4, 5, 6])
     @responses.activate
     def test_upload_length_deferred(self, chunk_size: int):
         upload_url = f"{self.client.url}test_upload_length_deferred"
-        
+
         responses.head(
             upload_url,
             adding_headers={"upload-offset": "0", "Upload-Defer-Length": "1"},
@@ -228,7 +274,7 @@ def test_upload_length_deferred(self, chunk_size: int):
         )
         self.assertTrue(uploader.upload_length_deferred)
         self.assertTrue(uploader.stop_at is None)
-        
+
         offset = 0
         while not (offset + chunk_size > 5):
             next_offset = min(offset + chunk_size, 5)
@@ -245,7 +291,7 @@ def test_upload_length_deferred(self, chunk_size: int):
             adding_headers={"upload-offset": "5"},
             match=[matchers.header_matcher(last_req_headers)],
         )
-        
+
         uploader.upload()
         self.assertEqual(uploader.offset, 5)
         self.assertEqual(uploader.stop_at, 5)

tusclient/uploader/baseuploader.py

@@ -238,6 +238,7 @@ def __init_url_and_offset(self, url: Optional[str] = None):
         If resumability is enabled, this would try to get the url from storage if available,
         otherwise it would request a new upload url from the tus server.
         """
+        key = None
         if url:
             self.set_url(url)
 
@@ -246,7 +247,21 @@ def __init_url_and_offset(self, url: Optional[str] = None):
             self.set_url(self.url_storage.get_item(key))
 
         if self.url:
-            self.offset = self.get_offset()
+            try:
+                self.offset = self.get_offset()
+            except TusCommunicationError as error:
+                # Special cases where url is still considered valid with given response code.
+                special_case_codes = [423]
+                # Process case where stored url is no longer valid.
+                if (
+                    key
+                    and 400 <= error.status_code <= 499
+                    and error.status_code not in special_case_codes
+                ):
+                    self.url = None
+                    self.url_storage.remove_item(key)
+                else:
+                    raise error
 
     def _get_fingerprint(self):
         with self.get_file_stream() as stream: