diff --git a/jf_agent/config_file_reader.py b/jf_agent/config_file_reader.py index 2aef477..f5744dd 100644 --- a/jf_agent/config_file_reader.py +++ b/jf_agent/config_file_reader.py @@ -558,6 +558,14 @@ def _make_datetimes_timezone_aware(datetime_str: str): skip_pulling_users = ADO_DEFAULT_API_URL not in base_url pull_from = _make_datetimes_timezone_aware(endpoint_git_instance_info['pull_from']) + + # Check if non-default backpopulation window should be used. + extra_kwargs = {} + if 'backpopulation_window_days' in endpoint_git_instance_info: + extra_kwargs['backpopulation_window_days'] = endpoint_git_instance_info[ + 'backpopulation_window_days' + ] + git_configs.append( JFIngestGitConfig( company_slug=company_slug, @@ -580,6 +588,7 @@ def _make_datetimes_timezone_aware(datetime_str: str): git_strip_text_content=agent_git_config.git_strip_text_content, check_ghc_mannequin_user_prs=agent_git_config.github_check_mannequin_users, skip_pulling_users=skip_pulling_users, + **extra_kwargs, ) ) diff --git a/tests/test_config_file.py b/tests/test_config_file.py index ef3c246..6abd415 100644 --- a/tests/test_config_file.py +++ b/tests/test_config_file.py @@ -1,7 +1,7 @@ import json import unittest from unittest import TestCase -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import yaml from jf_ingest.config import AzureDevopsAuthConfig as JFIngestAzureDevopsAuthConfig @@ -10,9 +10,52 @@ GitConfig, _get_git_config_from_yaml, _get_jf_ingest_git_auth_config, + get_ingest_config, ) +def _build_ado_git_config() -> GitConfig: + return GitConfig( + git_url='https://ado.com', + git_provider='ado', + git_instance_slug='ado-instance-1', + git_include_projects=[], + git_exclude_projects=[], + git_include_all_repos_inside_projects=[], + git_exclude_all_repos_inside_projects=[], + git_include_repos=[], + git_exclude_repos=[], + git_include_branches={}, + git_strip_text_content=False, + git_redact_names_and_urls=False, + gitlab_per_page_override=False, + git_verbose=False, + gitlab_keep_base_url=False, + creds_envvar_prefix='ORG1', + git_include_bbcloud_projects=[], + git_exclude_bbcloud_projects=[], + ) + + +def _build_ingest_config_inputs(endpoint_git_instance_info: dict): + config = MagicMock() + config.jira_url = None + config.git_configs = [_build_ado_git_config()] + config.skip_ssl_verification = False + config.run_mode_includes_send = False + config.jira_skip_saving_data_locally = False + config.outdir = 'agent-output-test' + config.jellyfish_api_base = 'https://api.jellyfish.co' + + creds = MagicMock() + creds.git_instance_to_creds = {'ado-instance-1': {'ado_token': 'token-1'}} + creds.jellyfish_api_token = 'jf-token' + + endpoint_git_instances_info = {'ado-instance-1': endpoint_git_instance_info} + + return config, creds, endpoint_git_instances_info + + class TestGitConfigGeneration(TestCase): def test_get_git_config_from_yaml_ado_default(self): ado_yaml_content = """ @@ -183,3 +226,82 @@ def test_get_jf_ingest_git_auth_config_convert_to_str(self): assert auth_config.token == 'token-2' assert auth_config.api_version == '7.0' assert auth_config.verify is True + + +class TestBackpopulationWindowDaysPassthrough(TestCase): + """ + Verifies the optional `backpopulation_window_days` value from the server endpoint + is forwarded into JFIngestGitConfig only when present, so that jf-ingest's default + applies otherwise. + """ + + @patch('jf_agent.config_file_reader.IngestionConfig') + @patch('jf_agent.config_file_reader.JFIngestGitConfig') + @patch('jf_agent.config_file_reader._get_jf_ingest_git_auth_config') + @patch('jf_agent.config_file_reader.get_company_info') + def test_backpopulation_window_days_forwarded_when_present( + self, + mock_get_company_info, + mock_get_auth, + mock_jf_ingest_git_config, + mock_ingestion_config, + ): + mock_get_company_info.return_value = {'company_slug': 'test-co'} + mock_get_auth.return_value = MagicMock() + + config, creds, endpoint_git_instances_info = _build_ingest_config_inputs( + endpoint_git_instance_info={ + 'slug': 'ado-instance-1', + 'key': 'ado-key', + 'repos_dict_v2': {}, + 'pull_from': '2024-01-01T00:00:00', + 'backpopulation_window_days': 90, + } + ) + + get_ingest_config( + config=config, + creds=creds, + endpoint_jira_info={}, + endpoint_git_instances_info=endpoint_git_instances_info, + jf_options={}, + ) + + self.assertEqual(mock_jf_ingest_git_config.call_count, 1) + kwargs = mock_jf_ingest_git_config.call_args.kwargs + self.assertEqual(kwargs.get('backpopulation_window_days'), 90) + + @patch('jf_agent.config_file_reader.IngestionConfig') + @patch('jf_agent.config_file_reader.JFIngestGitConfig') + @patch('jf_agent.config_file_reader._get_jf_ingest_git_auth_config') + @patch('jf_agent.config_file_reader.get_company_info') + def test_backpopulation_window_days_omitted_when_absent( + self, + mock_get_company_info, + mock_get_auth, + mock_jf_ingest_git_config, + mock_ingestion_config, + ): + mock_get_company_info.return_value = {'company_slug': 'test-co'} + mock_get_auth.return_value = MagicMock() + + config, creds, endpoint_git_instances_info = _build_ingest_config_inputs( + endpoint_git_instance_info={ + 'slug': 'ado-instance-1', + 'key': 'ado-key', + 'repos_dict_v2': {}, + 'pull_from': '2024-01-01T00:00:00', + } + ) + + get_ingest_config( + config=config, + creds=creds, + endpoint_jira_info={}, + endpoint_git_instances_info=endpoint_git_instances_info, + jf_options={}, + ) + + self.assertEqual(mock_jf_ingest_git_config.call_count, 1) + kwargs = mock_jf_ingest_git_config.call_args.kwargs + self.assertNotIn('backpopulation_window_days', kwargs)