Add configurable commit lookback

example.yml

@@ -208,6 +208,12 @@ git:
   #     - master
   #     - develop
 
+  # The number of days to look back when pulling commit data. Commits older than
+  # this many days will not be pulled. If omitted, defaults to 31.
+  # Reducing this value can significantly speed up agent runs for repositories
+  # with large amounts of commit history.
+  commit_lookback_days: 31
+
   # Strip out long-form text content (commit messages, PR text, etc).
   strip_text_content: False
 

jf_agent/config_file_reader.py

@@ -47,6 +47,7 @@ class GitConfig:
     # legacy fields ==================
     git_include_bbcloud_projects: List
     git_exclude_bbcloud_projects: List
+    git_commit_lookback_days: int = 31
     github_check_mannequin_users: bool = False
     # For ADO only
     ado_api_version: Optional[str] = None
@@ -697,6 +698,7 @@ def _get_git_config(git_config, git_provider_override=None, multiple=False) -> G
         git_redact_names_and_urls=git_config.get('redact_names_and_urls', False),
         gitlab_per_page_override=git_config.get('gitlab_per_page_override', None),
         git_verbose=git_config.get('verbose', False),
+        git_commit_lookback_days=git_config.get('commit_lookback_days', 31),
         creds_envvar_prefix=creds_envvar_prefix,
         gitlab_keep_base_url=git_config.get('keep_base_url', False),
         # ADO only

jf_agent/git/__init__.py

@@ -421,7 +421,9 @@
     }
 
 
-def pull_since_date_for_repo(instance_info, org_login, repo_id, commits_or_prs: str):
+def pull_since_date_for_repo(
+    instance_info, org_login, repo_id, commits_or_prs: str, commit_lookback_days: int = 31
+):
     assert commits_or_prs in ('commits', 'prs')
 
     instance_pull_from_dt = pytz.utc.localize(datetime.fromisoformat(instance_info['pull_from']))
@@ -438,8 +440,8 @@
             return instance_pull_from_dt
         else:
             if commits_or_prs == 'commits':
-                # We don't need to backpopulate the repo -- pull commits for last month
-                return pytz.utc.localize(datetime.utcnow() - timedelta(days=31))
+                # We don't need to backpopulate the repo -- pull recent commits
+                return pytz.utc.localize(datetime.utcnow() - timedelta(days=commit_lookback_days))
             else:
                 # We don't need to backpopulate the repo -- only need to pull PRs that have been updated
                 # more recently than PR with the latest update_date on the already-sent PRs

jf_agent/git/bitbucket_cloud_adapter.py

@@ -165,7 +165,11 @@ def get_commits_for_included_branches(
         for i, repo in enumerate(standardized_repos, start=1):
             with logging_helper.log_loop_iters('repo for branch commits', i, 1):
                 pull_since = pull_since_date_for_repo(
-                    server_git_instance_info, repo.project.login, repo.id, 'commits'
+                    server_git_instance_info,
+                    repo.project.login,
+                    repo.id,
+                    'commits',
+                    self.config.git_commit_lookback_days,
                 )
 
                 for branch in get_branches_for_standardized_repo(repo, included_branches):

jf_agent/git/bitbucket_server.py

@@ -82,6 +82,7 @@
                 endpoint_git_instance_info,
                 config.git_redact_names_and_urls,
                 config.git_verbose,
+                config.git_commit_lookback_days,
             ),
             item_id_dict_key='hash',
         )
@@ -258,6 +259,7 @@
     server_git_instance_info,
     redact_names_and_urls,
     verbose,
+    commit_lookback_days=31,
 ):
     for i, api_repo in enumerate(api_repos, start=1):
         with logging_helper.log_loop_iters('repo for branch commits', i, 1):
@@ -270,7 +272,11 @@
                 logger.info(f"Beginning download of commits for repo {repo}")
             api_project = client.projects[repo['project']['key']]
             pull_since = pull_since_date_for_repo(
-                server_git_instance_info, repo['project']['key'], repo['id'], 'commits'
+                server_git_instance_info,
+                repo['project']['key'],
+                repo['id'],
+                'commits',
+                commit_lookback_days,
             )
 
             # Determine branches to pull commits from for this repo. If no branches are explicitly

jf_agent/git/github.py

@@ -101,6 +101,7 @@
                 config.git_strip_text_content,
                 endpoint_git_instance_info,
                 config.git_redact_names_and_urls,
+                config.git_commit_lookback_days,
             ),
             item_id_dict_key='hash',
         )
@@ -281,11 +282,16 @@
     strip_text_content,
     server_git_instance_info,
     redact_names_and_urls,
+    commit_lookback_days=31,
 ):
     for i, repo in enumerate(api_repos, start=1):
         with logging_helper.log_loop_iters('repo for branch commits', i, 1):
             pull_since = pull_since_date_for_repo(
-                server_git_instance_info, repo['organization']['login'], repo['id'], 'commits'
+                server_git_instance_info,
+                repo['organization']['login'],
+                repo['id'],
+                'commits',
+                commit_lookback_days,
             )
 
             # Determine branches to pull commits from for this repo. If no branches are explicitly

jf_agent/git/github_gql_adapter.py

@@ -149,7 +149,11 @@ def get_repos(
                         # Get our internal 'pull_from' value
                         pull_since_for_commits = (
                             pull_since_date_for_repo(
-                                self.server_git_instance_info, nrm_project.login, repo_id, 'commits'
+                                self.server_git_instance_info,
+                                nrm_project.login,
+                                repo_id,
+                                'commits',
+                                self.config.git_commit_lookback_days,
                             )
                             or datetime.min
                         )
@@ -209,7 +213,11 @@ def get_commits_for_included_branches(
             with logging_helper.log_loop_iters('repo for branch commits', i, 1):
                 pull_since = (
                     pull_since_date_for_repo(
-                        server_git_instance_info, nrm_repo.project.login, nrm_repo.id, 'commits'
+                        server_git_instance_info,
+                        nrm_repo.project.login,
+                        nrm_repo.id,
+                        'commits',
+                        self.config.git_commit_lookback_days,
                     )
                     or datetime.min
                 )

jf_agent/git/gitlab_adapter.py

@@ -183,7 +183,11 @@ def get_commits_for_included_branches(
         for i, nrm_repo in enumerate(standardized_repos, start=1):
             with logging_helper.log_loop_iters('repo for branch commits', i, 1):
                 pull_since = pull_since_date_for_repo(
-                    server_git_instance_info, nrm_repo.project.login, nrm_repo.id, 'commits'
+                    server_git_instance_info,
+                    nrm_repo.project.login,
+                    nrm_repo.id,
+                    'commits',
+                    self.config.git_commit_lookback_days,
                 )
 
                 try:

tests/test_config_file.py

@@ -74,8 +74,27 @@
         assert git_config.git_provider == 'ado'
         assert git_config.git_url == 'https://ado.com'
         assert git_config.git_verbose is True
+        assert git_config.git_commit_lookback_days == 31
         assert git_config.ado_api_version is None  # Default value
 
+    def test_get_git_config_from_yaml_commit_lookback_days_override(self):
+        github_yaml_content = """
+        git:
+            provider: github
+            url: https://api.github.com
+            include_projects:
+              - example-org
+            commit_lookback_days: 7
+        """
+
+        yaml_config = yaml.safe_load(github_yaml_content)
+
+        git_configs: list[GitConfig] = _get_git_config_from_yaml(yaml_config)
+
+        self.assertEqual(len(git_configs), 1)
+        git_config = git_configs[0]
+        assert git_config.git_commit_lookback_days == 7
+
     def test_get_git_config_from_yaml_ado_version_override(self):
         ado_yaml_content = """
         git:
@@ -105,6 +124,7 @@
               url: https://ado.com
               verbose: true
               ado_api_version: '6.0'
+              commit_lookback_days: 14
             - provider: ado
               creds_envvar_prefix: ORG2
               instance_slug: ado-instance-2
@@ -125,13 +145,15 @@
         assert config_1.git_url == 'https://ado.com'
         assert config_1.git_verbose is True
         assert config_1.ado_api_version == '6.0'
+        assert config_1.git_commit_lookback_days == 14
 
         config_2 = git_configs[1]
         assert config_2.git_instance_slug == 'ado-instance-2'
         assert config_2.git_provider == 'ado'
         assert config_2.git_url == 'https://ado.com'
         assert config_2.git_verbose is True
         assert config_2.ado_api_version is None  # Default value
+        assert config_2.git_commit_lookback_days == 31
 
     def test_get_jf_ingest_git_auth_config(self):
         ado_yaml_content = """