Merge pull request #174 from codecov/dana/file-fix

use --file option to let users set specific files to upload

Author: dana-yaish

codecov_cli/commands/upload.py

@@ -64,11 +64,17 @@ def _turn_env_vars_into_dict(ctx, params, value):
     "--file",
     "--coverage-files-search-direct-file",
     "coverage_files_search_explicitly_listed_files",
-    help="Explicit files to upload",
+    help="Explicit files to upload. These will be added to the coverage files found for upload. If you wish to only upload the specified files, please consider using --disable-search to disable uploading other files.",
     type=click.Path(path_type=pathlib.Path),
     multiple=True,
     default=[],
 )
+@click.option(
+    "--disable-search",
+    help="Disable search for coverage files. This is helpful when specifying what files you want to uload with the --file option.",
+    is_flag=True,
+    default=False,
+)
 @click.option(
     "-b",
     "--build",
@@ -189,6 +195,7 @@ def do_upload(
     coverage_files_search_root_folder: pathlib.Path,
     coverage_files_search_exclude_folders: typing.List[pathlib.Path],
     coverage_files_search_explicitly_listed_files: typing.List[pathlib.Path],
+    disable_search: bool,
     token: typing.Optional[uuid.UUID],
     plugin_names: typing.List[str],
     branch: typing.Optional[str],
@@ -227,6 +234,7 @@ def do_upload(
                 pull_request_number=pull_request_number,
                 git_service=git_service,
                 enterprise_url=enterprise_url,
+                disable_search=disable_search,
             )
         ),
     )
@@ -260,4 +268,5 @@ def do_upload(
         dry_run=dry_run,
         git_service=git_service,
         enterprise_url=enterprise_url,
+        disable_search=disable_search,
     )

codecov_cli/services/upload/__init__.py

@@ -47,12 +47,14 @@ def do_upload_logic(
     dry_run: bool = False,
     git_service: typing.Optional[str],
     enterprise_url: typing.Optional[str],
+    disable_search: bool = False,
 ):
     preparation_plugins = select_preparation_plugins(cli_config, plugin_names)
     coverage_file_selector = select_coverage_file_finder(
         coverage_files_search_root_folder,
         coverage_files_search_exclude_folders,
         coverage_files_search_explicitly_listed_files,
+        disable_search,
     )
     network_finder = select_network_finder(versioning_system)
     collector = UploadCollector(

codecov_cli/services/upload/coverage_file_finder.py

@@ -1,10 +1,13 @@
+import logging
 import os
 import typing
 from pathlib import Path
 
 from codecov_cli.helpers.folder_searcher import globs_to_regex, search_files
 from codecov_cli.types import UploadCollectionResultFile
 
+logger = logging.getLogger("codecovcli")
+
 coverage_files_patterns = [
     "*.clover",
     "*.codecov.*",
@@ -163,26 +166,93 @@
 
 class CoverageFileFinder(object):
     def __init__(
-        self, project_root: Path = None, folders_to_ignore: typing.List[str] = None
+        self,
+        project_root: Path = None,
+        folders_to_ignore: typing.List[str] = None,
+        explicitly_listed_files: typing.List[Path] = None,
+        disable_search: bool = False,
     ):
         self.project_root = project_root or Path(os.getcwd())
         self.folders_to_ignore = folders_to_ignore or []
+        self.explicitly_listed_files = explicitly_listed_files or None
+        self.disable_search = disable_search
 
     def find_coverage_files(self) -> typing.List[UploadCollectionResultFile]:
-        regex_patterns_to_include = globs_to_regex(coverage_files_patterns)
         regex_patterns_to_exclude = globs_to_regex(coverage_files_excluded_patterns)
+        coverage_files_paths = []
+        user_coverage_files_paths = []
+        if self.explicitly_listed_files:
+            user_coverage_files_paths = self.get_user_specified_coverage_files(
+                regex_patterns_to_exclude
+            )
+        if not self.disable_search:
+            regex_patterns_to_include = globs_to_regex(coverage_files_patterns)
+            coverage_files_paths = search_files(
+                self.project_root,
+                default_folders_to_ignore + self.folders_to_ignore,
+                filename_include_regex=regex_patterns_to_include,
+                filename_exclude_regex=regex_patterns_to_exclude,
+            )
+        result_files = [
+            UploadCollectionResultFile(path)
+            for path in coverage_files_paths
+            if coverage_files_paths
+        ]
+        user_result_files = [
+            UploadCollectionResultFile(path)
+            for path in user_coverage_files_paths
+            if user_coverage_files_paths
+        ]
+
+        return list(set(result_files + user_result_files))
 
-        coverage_files_paths = search_files(
-            self.project_root,
-            default_folders_to_ignore + self.folders_to_ignore,
-            filename_include_regex=regex_patterns_to_include,
-            filename_exclude_regex=regex_patterns_to_exclude,
+    def get_user_specified_coverage_files(self, regex_patterns_to_exclude):
+        user_filenames_to_include = []
+        files_excluded_but_user_includes = []
+        for file in self.explicitly_listed_files:
+            user_filenames_to_include.append(file.name)
+            if regex_patterns_to_exclude.match(file.name):
+                files_excluded_but_user_includes.append(str(file))
+        if files_excluded_but_user_includes:
+            logger.warning(
+                "Some files being explicitly added are found in the list of excluded files for upload.",
+                extra=dict(
+                    extra_log_attributes=dict(files=files_excluded_but_user_includes)
+                ),
+            )
+        regex_patterns_to_include = globs_to_regex(user_filenames_to_include)
+        multipart_include_regex = globs_to_regex(
+            [str(path.resolve()) for path in self.explicitly_listed_files]
         )
+        user_coverage_files_paths = list(
+            search_files(
+                self.project_root,
+                default_folders_to_ignore + self.folders_to_ignore,
+                filename_include_regex=regex_patterns_to_include,
+                filename_exclude_regex=regex_patterns_to_exclude,
+                multipart_include_regex=multipart_include_regex,
+            )
+        )
+        not_found_files = []
+        for filepath in self.explicitly_listed_files:
+            if filepath.resolve() not in user_coverage_files_paths:
+                not_found_files.append(filepath)
+
+        if not_found_files:
+            logger.warning(
+                "Some files were not found",
+                extra=dict(extra_log_attributes=dict(not_found_files=not_found_files)),
+            )
 
-        return [UploadCollectionResultFile(path) for path in coverage_files_paths]
+        return user_coverage_files_paths
 
 
 def select_coverage_file_finder(
-    root_folder_to_search, folders_to_ignore, explicitly_listed_files
+    root_folder_to_search, folders_to_ignore, explicitly_listed_files, disable_search
 ):
-    return CoverageFileFinder(root_folder_to_search, folders_to_ignore)
+    return CoverageFileFinder(
+        root_folder_to_search,
+        folders_to_ignore,
+        explicitly_listed_files,
+        disable_search,
+    )

tests/services/upload/test_coverage_file_finder.py

@@ -23,11 +23,17 @@ def test_find_coverage_files_mocked_search_files(self, mocker):
         )
 
         expected = [
-            UploadCollectionResultFile(Path("a/b.txt")),
             UploadCollectionResultFile(Path("c.txt")),
+            UploadCollectionResultFile(Path("a/b.txt")),
         ]
 
-        assert CoverageFileFinder().find_coverage_files() == expected
+        expected_paths = sorted([file.get_filename() for file in expected])
+
+        actual_paths = sorted(
+            [file.get_filename() for file in CoverageFileFinder().find_coverage_files()]
+        )
+
+        assert expected_paths == actual_paths
 
     def test_find_coverage_files(self, tmp_path):
         (tmp_path / "sub").mkdir()

tests/services/upload/test_upload_service.py

@@ -79,7 +79,7 @@ def test_do_upload_logic_happy_path_legacy_uploader(mocker):
     mock_select_preparation_plugins.assert_called_with(
         cli_config, ["first_plugin", "another", "forth"]
     )
-    mock_select_coverage_file_finder.assert_called_with(None, None, None)
+    mock_select_coverage_file_finder.assert_called_with(None, None, None, False)
     mock_select_network_finder.assert_called_with(versioning_system)
     mock_generate_upload_data.assert_called_with()
     mock_send_upload_data.assert_called_with(
@@ -164,7 +164,7 @@ def test_do_upload_logic_happy_path(mocker):
     mock_select_preparation_plugins.assert_called_with(
         cli_config, ["first_plugin", "another", "forth"]
     )
-    mock_select_coverage_file_finder.assert_called_with(None, None, None)
+    mock_select_coverage_file_finder.assert_called_with(None, None, None, False)
     mock_select_network_finder.assert_called_with(versioning_system)
     mock_generate_upload_data.assert_called_with()
     mock_send_upload_data.assert_called_with(
@@ -236,7 +236,7 @@ def test_do_upload_logic_dry_run(mocker):
             enterprise_url=None,
         )
     out_bytes = parse_outstreams_into_log_lines(outstreams[0].getvalue())
-    mock_select_coverage_file_finder.assert_called_with(None, None, None)
+    mock_select_coverage_file_finder.assert_called_with(None, None, None, False)
     mock_select_network_finder.assert_called_with(versioning_system)
     assert mock_generate_upload_data.call_count == 1
     assert mock_send_upload_data.call_count == 0