Merge pull request #639 from Aiven-Open/rdunklau/fix_timeline_disappearing

One more occurence of making pg_wal contents disappear

Author: packi

pghoard/webserver.py

@@ -537,7 +537,7 @@ def get_wal_or_timeline_file(self, site: str, filename: str, filetype: str) -> N
         # After reaching a recovery_target and restart of a PG server, PG wants to replay and refetch
         # files from the archive starting from the latest checkpoint. We have potentially fetched these files
         # already earlier. Check if we have the files already and if we do, don't go over the network to refetch
-        # them yet again but just rename them to the path that PG is requesting.
+        # them yet again but just copy them to the path that PG is requesting.
         xlog_path = os.path.join(xlog_dir, filename)
         exists_on_disk = os.path.exists(xlog_path)
         if exists_on_disk and filename not in self.server.served_from_disk:
@@ -550,7 +550,8 @@ def get_wal_or_timeline_file(self, site: str, filename: str, filetype: str) -> N
             except ValueError as e:
                 self.server.log.warning("Found file: %r but it was invalid: %s", xlog_path, e)
             else:
-                self._rename(xlog_path, target_path)
+                if xlog_path != target_path:
+                    shutil.copyfile(xlog_path, target_path)
                 self.server.served_from_disk.append(filename)
                 self.server.most_recently_served_files[filetype] = {
                     "name": filename,

test/test_webserver.py

@@ -189,6 +189,7 @@ def test_wal_fetch_optimization(self, pghoard):
         )
         assert os.path.exists(output_path)
         os.unlink(output_path)
+        os.unlink(valid_wal_path)
 
         with pytest.raises(postgres_command.PGCError):
             restore_command(
@@ -601,8 +602,11 @@ def test_retry_fetches_remote(self, pghoard_no_mp):
         status = conn.getresponse().status
         assert status == 201
 
+        # This test used to ensure we deleted files from the pg_xlog directory
+        # It now tests that if the operation results in a copy, the original file
+        # is still copied instead of being removed
         with open(storage_name, "rb") as f:
-            assert f.read() == storage_data
+            assert f.read() == on_disk_data
 
     def test_restore_command_retry(self, pghoard):
         failures = [0, ""]