diff --git a/example.yml b/example.yml index 5798378..dfedda8 100644 --- a/example.yml +++ b/example.yml @@ -208,6 +208,12 @@ git: # - master # - develop + # The number of days to look back when pulling commit data. Commits older than + # this many days will not be pulled. If omitted, defaults to 31. + # Reducing this value can significantly speed up agent runs for repositories + # with large amounts of commit history. + commit_lookback_days: 31 + # Strip out long-form text content (commit messages, PR text, etc). strip_text_content: False diff --git a/jf_agent/config_file_reader.py b/jf_agent/config_file_reader.py index f5744dd..034cfa3 100644 --- a/jf_agent/config_file_reader.py +++ b/jf_agent/config_file_reader.py @@ -47,6 +47,7 @@ class GitConfig: # legacy fields ================== git_include_bbcloud_projects: List git_exclude_bbcloud_projects: List + git_commit_lookback_days: int = 31 github_check_mannequin_users: bool = False # For ADO only ado_api_version: Optional[str] = None @@ -697,6 +698,7 @@ def _get_git_config(git_config, git_provider_override=None, multiple=False) -> G git_redact_names_and_urls=git_config.get('redact_names_and_urls', False), gitlab_per_page_override=git_config.get('gitlab_per_page_override', None), git_verbose=git_config.get('verbose', False), + git_commit_lookback_days=git_config.get('commit_lookback_days', 31), creds_envvar_prefix=creds_envvar_prefix, gitlab_keep_base_url=git_config.get('keep_base_url', False), # ADO only diff --git a/jf_agent/git/__init__.py b/jf_agent/git/__init__.py index d4afa3a..3bd26c8 100644 --- a/jf_agent/git/__init__.py +++ b/jf_agent/git/__init__.py @@ -421,7 +421,9 @@ def load_and_dump_git( } -def pull_since_date_for_repo(instance_info, org_login, repo_id, commits_or_prs: str): +def pull_since_date_for_repo( + instance_info, org_login, repo_id, commits_or_prs: str, commit_lookback_days: int = 31 +): assert commits_or_prs in ('commits', 'prs') instance_pull_from_dt = pytz.utc.localize(datetime.fromisoformat(instance_info['pull_from'])) @@ -438,8 +440,8 @@ def pull_since_date_for_repo(instance_info, org_login, repo_id, commits_or_prs: return instance_pull_from_dt else: if commits_or_prs == 'commits': - # We don't need to backpopulate the repo -- pull commits for last month - return pytz.utc.localize(datetime.utcnow() - timedelta(days=31)) + # We don't need to backpopulate the repo -- pull recent commits + return pytz.utc.localize(datetime.utcnow() - timedelta(days=commit_lookback_days)) else: # We don't need to backpopulate the repo -- only need to pull PRs that have been updated # more recently than PR with the latest update_date on the already-sent PRs diff --git a/jf_agent/git/bitbucket_cloud_adapter.py b/jf_agent/git/bitbucket_cloud_adapter.py index ff28e63..73ac8a2 100644 --- a/jf_agent/git/bitbucket_cloud_adapter.py +++ b/jf_agent/git/bitbucket_cloud_adapter.py @@ -165,7 +165,11 @@ def get_commits_for_included_branches( for i, repo in enumerate(standardized_repos, start=1): with logging_helper.log_loop_iters('repo for branch commits', i, 1): pull_since = pull_since_date_for_repo( - server_git_instance_info, repo.project.login, repo.id, 'commits' + server_git_instance_info, + repo.project.login, + repo.id, + 'commits', + self.config.git_commit_lookback_days, ) for branch in get_branches_for_standardized_repo(repo, included_branches): diff --git a/jf_agent/git/bitbucket_server.py b/jf_agent/git/bitbucket_server.py index 2a5d0f8..2b00f60 100644 --- a/jf_agent/git/bitbucket_server.py +++ b/jf_agent/git/bitbucket_server.py @@ -82,6 +82,7 @@ def download_and_write_commits(): endpoint_git_instance_info, config.git_redact_names_and_urls, config.git_verbose, + config.git_commit_lookback_days, ), item_id_dict_key='hash', ) @@ -258,6 +259,7 @@ def get_commits_for_included_branches( server_git_instance_info, redact_names_and_urls, verbose, + commit_lookback_days=31, ): for i, api_repo in enumerate(api_repos, start=1): with logging_helper.log_loop_iters('repo for branch commits', i, 1): @@ -270,7 +272,11 @@ def get_commits_for_included_branches( logger.info(f"Beginning download of commits for repo {repo}") api_project = client.projects[repo['project']['key']] pull_since = pull_since_date_for_repo( - server_git_instance_info, repo['project']['key'], repo['id'], 'commits' + server_git_instance_info, + repo['project']['key'], + repo['id'], + 'commits', + commit_lookback_days, ) # Determine branches to pull commits from for this repo. If no branches are explicitly diff --git a/jf_agent/git/github.py b/jf_agent/git/github.py index b158931..dc45cbb 100644 --- a/jf_agent/git/github.py +++ b/jf_agent/git/github.py @@ -101,6 +101,7 @@ def download_and_write_commits(): config.git_strip_text_content, endpoint_git_instance_info, config.git_redact_names_and_urls, + config.git_commit_lookback_days, ), item_id_dict_key='hash', ) @@ -281,11 +282,16 @@ def get_commits_for_included_branches( strip_text_content, server_git_instance_info, redact_names_and_urls, + commit_lookback_days=31, ): for i, repo in enumerate(api_repos, start=1): with logging_helper.log_loop_iters('repo for branch commits', i, 1): pull_since = pull_since_date_for_repo( - server_git_instance_info, repo['organization']['login'], repo['id'], 'commits' + server_git_instance_info, + repo['organization']['login'], + repo['id'], + 'commits', + commit_lookback_days, ) # Determine branches to pull commits from for this repo. If no branches are explicitly diff --git a/jf_agent/git/github_gql_adapter.py b/jf_agent/git/github_gql_adapter.py index 83c239e..84f1d8a 100644 --- a/jf_agent/git/github_gql_adapter.py +++ b/jf_agent/git/github_gql_adapter.py @@ -149,7 +149,11 @@ def get_repos( # Get our internal 'pull_from' value pull_since_for_commits = ( pull_since_date_for_repo( - self.server_git_instance_info, nrm_project.login, repo_id, 'commits' + self.server_git_instance_info, + nrm_project.login, + repo_id, + 'commits', + self.config.git_commit_lookback_days, ) or datetime.min ) @@ -209,7 +213,11 @@ def get_commits_for_included_branches( with logging_helper.log_loop_iters('repo for branch commits', i, 1): pull_since = ( pull_since_date_for_repo( - server_git_instance_info, nrm_repo.project.login, nrm_repo.id, 'commits' + server_git_instance_info, + nrm_repo.project.login, + nrm_repo.id, + 'commits', + self.config.git_commit_lookback_days, ) or datetime.min ) diff --git a/jf_agent/git/gitlab_adapter.py b/jf_agent/git/gitlab_adapter.py index 5fd823a..3317a11 100644 --- a/jf_agent/git/gitlab_adapter.py +++ b/jf_agent/git/gitlab_adapter.py @@ -183,7 +183,11 @@ def get_commits_for_included_branches( for i, nrm_repo in enumerate(standardized_repos, start=1): with logging_helper.log_loop_iters('repo for branch commits', i, 1): pull_since = pull_since_date_for_repo( - server_git_instance_info, nrm_repo.project.login, nrm_repo.id, 'commits' + server_git_instance_info, + nrm_repo.project.login, + nrm_repo.id, + 'commits', + self.config.git_commit_lookback_days, ) try: diff --git a/tests/test_config_file.py b/tests/test_config_file.py index 6abd415..8d58622 100644 --- a/tests/test_config_file.py +++ b/tests/test_config_file.py @@ -74,8 +74,27 @@ def test_get_git_config_from_yaml_ado_default(self): assert git_config.git_provider == 'ado' assert git_config.git_url == 'https://ado.com' assert git_config.git_verbose is True + assert git_config.git_commit_lookback_days == 31 assert git_config.ado_api_version is None # Default value + def test_get_git_config_from_yaml_commit_lookback_days_override(self): + github_yaml_content = """ + git: + provider: github + url: https://api.github.com + include_projects: + - example-org + commit_lookback_days: 7 + """ + + yaml_config = yaml.safe_load(github_yaml_content) + + git_configs: list[GitConfig] = _get_git_config_from_yaml(yaml_config) + + self.assertEqual(len(git_configs), 1) + git_config = git_configs[0] + assert git_config.git_commit_lookback_days == 7 + def test_get_git_config_from_yaml_ado_version_override(self): ado_yaml_content = """ git: @@ -105,6 +124,7 @@ def test_get_git_config_multi_provider(self): url: https://ado.com verbose: true ado_api_version: '6.0' + commit_lookback_days: 14 - provider: ado creds_envvar_prefix: ORG2 instance_slug: ado-instance-2 @@ -125,6 +145,7 @@ def test_get_git_config_multi_provider(self): assert config_1.git_url == 'https://ado.com' assert config_1.git_verbose is True assert config_1.ado_api_version == '6.0' + assert config_1.git_commit_lookback_days == 14 config_2 = git_configs[1] assert config_2.git_instance_slug == 'ado-instance-2' @@ -132,6 +153,7 @@ def test_get_git_config_multi_provider(self): assert config_2.git_url == 'https://ado.com' assert config_2.git_verbose is True assert config_2.ado_api_version is None # Default value + assert config_2.git_commit_lookback_days == 31 def test_get_jf_ingest_git_auth_config(self): ado_yaml_content = """