Merge pull request #220 from hc-sc-ocdo-bdpd/hashing

Hashing and copy method on file class

Author: BrennanHCSC

file_processing/file.py

@@ -8,6 +8,7 @@
     NotTranscriptionApplicableError,
     OptionalDependencyNotInstalledError
 )
+import shutil
 
 
 class File:
@@ -151,10 +152,30 @@ def save(self, output_path: str = None) -> None:
         Saves the processed file to the specified output path.
 
         Args:
-            output_path (str): The destination path for saving the processed file.
+            output_path (str): Destination path (defaults to original file path).
         """
         self.processor.save(output_path)
 
+    def copy(self, output_path: str, verify_integrity: bool = False) -> None:
+        """
+        Copies the original file bit-for-bit.
+
+        Args:
+            output_path (str): Destination path (defaults to original file path).
+            verify_integrity (bool): If True, re-compute and compare hash after copy.
+        """
+        original_hash = self.hash if verify_integrity else None
+        src = self.processor.file_path
+        shutil.copy2(str(src), str(output_path))
+
+        if verify_integrity:
+            copied = File(str(output_path), open_file=False)
+            if original_hash != copied.hash:
+                raise FileProcessingFailedError(
+                    f"Integrity check failed on copy: {output_path} hash changed "
+                    f"({original_hash} → {copied.hash})"
+                )
+
     def process(self) -> None:
         """
         Executes the processing operation on the file.
@@ -183,23 +204,23 @@ def owner(self) -> str:
         return self.processor.owner
 
     @property
-    def size(self) -> str:
-        """str: Returns the size of the file in bytes."""
+    def size(self) -> int:
+        """int: Returns the size of the file in bytes."""
         return self.processor.size
 
     @property
-    def modification_time(self) -> str:
-        """str: Returns the last modification time of the file."""
+    def modification_time(self) -> float:
+        """float: Returns the last modification time of the file."""
         return self.processor.modification_time
 
     @property
-    def access_time(self) -> str:
-        """str: Returns the last access time of the file."""
+    def access_time(self) -> float:
+        """float: Returns the last access time of the file."""
         return self.processor.access_time
 
     @property
-    def creation_time(self) -> str:
-        """str: Returns the creation time of the file."""
+    def creation_time(self) -> float:
+        """float: Returns the creation time of the file."""
         return self.processor.creation_time
 
     @property
@@ -226,6 +247,11 @@ def is_symlink(self) -> bool:
     def absolute_path(self) -> str:
         """str: Returns the absolute path of the file."""
         return self.processor.absolute_path
+    
+    @property
+    def hash(self) -> str:
+        """str: Hex digest of this file's content (default SHA256)."""
+        return self.processor.hash
 
     @property
     def metadata(self) -> dict:

file_processing/file_processor_strategy.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 import sys
 import importlib.util
+from hashlib import md5, sha256
 from file_processing.errors import FileProcessingFailedError
 
 class FileProcessorStrategy(ABC):
@@ -77,6 +78,47 @@ def _find_owner(self, file_path: str) -> str:
             return f'{domain}/{name}'
         return ''
 
+    @property
+    def hash(self) -> str:
+        """
+        Computes (if necessary) and returns the hash of the file content.
+
+        Returns:
+            str: Hexadecimal hash of the file.
+        """
+        if not hasattr(self, '_hash'):
+            self._hash = self.compute_hash()
+        return self._hash
+
+    def compute_hash(self, algorithm: str = 'sha256') -> str:
+        """
+        Computes the hash of the file using the specified algorithm.
+
+        Args:
+            algorithm (str, optional): The hashing algorithm to use ('sha256' or 'md5'). Defaults to 'sha256'.
+
+        Returns:
+            str: Hexadecimal hash of the file.
+
+        Raises:
+            ValueError: If an unsupported algorithm is specified.
+            FileProcessingFailedError: If file hashing fails.
+        """
+        hash_func = {'md5': md5, 'sha256': sha256}.get(algorithm)
+        if not hash_func:
+            raise ValueError(f"Unsupported hash algorithm: {algorithm}")
+
+        hasher = hash_func()
+        try:
+            with open(self.file_path, 'rb') as f:
+                for chunk in iter(lambda: f.read(8192), b''):
+                    hasher.update(chunk)
+            return hasher.hexdigest()
+        except Exception as e:
+            raise FileProcessingFailedError(
+                f"Error computing hash for {self.file_path}: {e}"
+            )
+
     @abstractmethod
     def process(self) -> None:
         """
@@ -95,4 +137,5 @@ def save(self) -> None:
         """
         Saves the processed file after any metadata or content modifications.
         This method must be implemented by subclasses to define save behavior.
-        """
+        """
+        pass

tests/unit/test_audio_processor.py

@@ -1,5 +1,6 @@
 from unittest.mock import patch
 from pathlib import Path
+import shutil
 import pytest
 from mutagen import File as MutagenFile
 from mutagen.easyid3 import EasyID3
@@ -68,7 +69,6 @@ def test_save_audio_metadata(copy_file, bitrate, length):
 def test_change_audio_artist_title_date(copy_file, bitrate, length):
     audio_file = MutagenFile(copy_file)
     if isinstance(audio_file, (MP3, FLAC, OggVorbis, MP4)):
-        # Change metadata via Document object
         audio_file = MutagenFile(copy_file)
         if isinstance(audio_file, MP3):
             audio_file = EasyID3(copy_file)
@@ -86,7 +86,6 @@ def test_change_audio_artist_title_date(copy_file, bitrate, length):
             audio_file.tags['\xa9day'] = "2023-11-22"
             audio_file.tags['\xa9nam'] = "New Title"
             audio_file.tags['\xa9wrk'] = "Health Canada"
-        # Save the file
         audio_file.save()
         test_audio_metadata(copy_file, bitrate, length, 'New Artist',
                             '2023-11-22', 'New Title', 'Health Canada')
@@ -101,6 +100,7 @@ def test_not_opening_file(path, bitrate, length, artist, date, title, organizati
         File(path, open_file=False)
         mock_open.assert_not_called()
 
+
 invalid_save_locations = [
     (test_files_path / 'sample_speech.mp3', '/non_existent_folder/sample_speech.mp3')
 ]
@@ -122,3 +122,31 @@ def test_audio_invalid_save_location(path, save_path):
 def test_audio_corrupted_file_processing(path):
     with pytest.raises(FileProcessingFailedError):
         File(path)
+
+
+@pytest.mark.parametrize("path", [v[0] for v in values])
+@pytest.mark.parametrize("algorithm", ["md5", "sha256"])
+def test_audio_copy_with_integrity(path, algorithm, tmp_path):
+    file_obj = File(path, open_file=False)
+    original_hash = file_obj.processor.compute_hash(algorithm)
+
+    dest_path = tmp_path / Path(path).name
+    file_obj.copy(str(dest_path), verify_integrity=True)
+
+    copied = File(str(dest_path))
+    assert copied.processor.compute_hash(algorithm) == original_hash
+
+
+@pytest.mark.parametrize("path", [v[0] for v in values])
+def test_audio_copy_integrity_failure(path, tmp_path, monkeypatch):
+    file_obj = File(path, open_file=False)
+
+    def corrupt(src, dest, *, follow_symlinks=True):
+        with open(dest, 'w') as f:
+            f.write("CORRUPTED!")
+
+    monkeypatch.setattr(shutil, "copy2", corrupt)
+
+    with pytest.raises(FileProcessingFailedError) as excinfo:
+        file_obj.copy(str(tmp_path / Path(path).name), verify_integrity=True)
+    assert "Integrity check failed" in str(excinfo.value)

tests/unit/test_cpp_processor.py

@@ -1,5 +1,7 @@
 import os
+import shutil
 import pytest
+from pathlib import Path
 from unittest.mock import patch
 from file_processing.file import File
 from file_processing.errors import FileProcessingFailedError
@@ -44,4 +46,31 @@ def test_cpp_processor_open_file_false(file_name):
     cpp_file_path = test_files_path / file_name
     with patch("builtins.open") as mock_open:
         File(str(cpp_file_path), open_file=False)
-        mock_open.assert_not_called()
+        mock_open.assert_not_called()
+
+@pytest.mark.parametrize("file_name", [v[0] for v in values])
+@pytest.mark.parametrize("algorithm", ["md5", "sha256"])
+def test_cpp_copy_with_integrity(file_name, algorithm, tmp_path):
+    src_path = test_files_path / file_name
+    file_obj = File(str(src_path), open_file=False)
+    original_hash = file_obj.processor.compute_hash(algorithm)
+
+    dest_path = tmp_path / file_name
+    file_obj.copy(str(dest_path), verify_integrity=True)
+
+    copied_file = File(str(dest_path))
+    assert copied_file.processor.compute_hash(algorithm) == original_hash
+
+@pytest.mark.parametrize("file_name", [v[0] for v in values])
+def test_cpp_copy_integrity_failure(file_name, tmp_path, monkeypatch):
+    file_obj = File(str(test_files_path / file_name), open_file=False)
+
+    def corrupt(src, dest, *, follow_symlinks=True):
+        with open(dest, 'w') as f:
+            f.write("CORRUPTED!")
+
+    monkeypatch.setattr(shutil, "copy2", corrupt)
+
+    with pytest.raises(FileProcessingFailedError) as excinfo:
+        file_obj.copy(str(tmp_path / file_name), verify_integrity=True)
+    assert "Integrity check failed" in str(excinfo.value)

tests/unit/test_csv_processor.py

@@ -1,19 +1,21 @@
 import os
+from pathlib import Path
 from unittest.mock import patch
 import pytest
 from file_processing import File
 from file_processing.errors import FileProcessingFailedError
 from file_processing_test_data import get_test_files_path
 
+# Data fixture
 test_files_path = get_test_files_path()
-
-variable_names = "path, text_length, encoding, num_rows, num_cols, num_cells, empty_cells"
+variable_names = (
+    "path, text_length, encoding, num_rows, num_cols, num_cells, empty_cells"
+)
 values = [
-   (test_files_path / '2021_Census_English.csv', 6084302, 'ISO-8859-1', 36835, 23, 847205, 253932),
-   (test_files_path / 'Approved_Schools_2023_10_01.csv', 1403268, 'UTF-8-SIG', 5385, 13, 70005, 73)
+    (test_files_path / '2021_Census_English.csv', 6084302, 'ISO-8859-1', 36835, 23, 847205, 253932),
+    (test_files_path / 'Approved_Schools_2023_10_01.csv', 1403268, 'UTF-8-SIG', 5385, 13, 70005, 73)
 ]
 
-
 @pytest.mark.parametrize(variable_names, values)
 def test_csv_metadata(path, text_length, encoding, num_rows, num_cols, num_cells, empty_cells):
     file_obj = File(path)
@@ -24,22 +26,58 @@ def test_csv_metadata(path, text_length, encoding, num_rows, num_cols, num_cells
     assert file_obj.metadata['num_cells'] == num_cells
     assert file_obj.metadata['empty_cells'] == empty_cells
 
-
 @pytest.mark.parametrize(variable_names, values)
 def test_save_csv_metadata(copy_file, text_length, encoding, num_rows, num_cols, num_cells, empty_cells):
+    # Reuse metadata assertions on the saved file
     test_csv_metadata(copy_file, text_length, encoding, num_rows, num_cols, num_cells, empty_cells)
 
-
-@pytest.mark.parametrize("valid_path", [path for path, *_ in values])
+@pytest.mark.parametrize("valid_path", [p for p, *_ in values])
 def test_csv_invalid_save_location(valid_path):
     csv_file = File(valid_path)
-    invalid_save_path = '/non_existent_folder/' + os.path.basename(valid_path)
+    invalid_path = '/non_existent_folder/' + os.path.basename(str(valid_path))
     with pytest.raises(FileProcessingFailedError):
-        csv_file.processor.save(invalid_save_path)
-
+        csv_file.processor.save(invalid_path)
 
 @pytest.mark.parametrize(variable_names, values)
 def test_not_opening_file(path, text_length, encoding, num_rows, num_cols, num_cells, empty_cells):
     with patch('builtins.open', autospec=True) as mock_open:
         File(path, open_file=False)
         mock_open.assert_not_called()
+
+@pytest.mark.parametrize(variable_names, values)
+@pytest.mark.parametrize("algorithm", ["md5", "sha256"])
+def test_csv_copy_with_integrity(path, tmp_path, algorithm,
+                                 text_length, encoding, num_rows, num_cols, num_cells, empty_cells):
+    # 1. Compute original hash (raw file)
+    file_obj = File(path, open_file=False)
+    original_hash = file_obj.processor.compute_hash(algorithm)
+
+    # 2. Copy with integrity verification
+    dest = tmp_path / Path(path).name
+    file_obj.copy(str(dest), verify_integrity=True)
+
+    # 3. Re-open and verify metadata unchanged
+    saved = File(str(dest))
+    assert len(saved.metadata['text']) == text_length
+    assert saved.metadata['encoding'] == encoding
+    assert saved.metadata['num_rows'] == num_rows
+    assert saved.metadata['num_cols'] == num_cols
+    assert saved.metadata['num_cells'] == num_cells
+    assert saved.metadata['empty_cells'] == empty_cells
+
+    # 4. Verify hash matches for chosen algorithm
+    assert saved.processor.compute_hash(algorithm) == original_hash
+
+@pytest.mark.parametrize("path", [p for p, *_ in values])
+def test_csv_copy_integrity_failure(path, tmp_path, monkeypatch):
+    file_obj = File(path, open_file=False)
+    # Simulate corruption in raw copy
+    import shutil
+    def corrupt(src, dest, *, follow_symlinks=True):
+        with open(dest, 'w') as f:
+            f.write('CORRUPTED!')
+    monkeypatch.setattr(shutil, 'copy2', corrupt)
+
+    with pytest.raises(FileProcessingFailedError) as excinfo:
+        file_obj.copy(str(tmp_path / Path(path).name), verify_integrity=True)
+    assert 'Integrity check failed' in str(excinfo.value)