Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions jf_agent/config_file_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,14 @@ def _make_datetimes_timezone_aware(datetime_str: str):
skip_pulling_users = ADO_DEFAULT_API_URL not in base_url

pull_from = _make_datetimes_timezone_aware(endpoint_git_instance_info['pull_from'])

# Check if non-default backpopulation window should be used.
extra_kwargs = {}
if 'backpopulation_window_days' in endpoint_git_instance_info:
extra_kwargs['backpopulation_window_days'] = endpoint_git_instance_info[
'backpopulation_window_days'
]
Comment on lines 560 to +567

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added unit coverage in tests/test_config_file.py (TestBackpopulationWindowDaysPassthrough): one test asserts backpopulation_window_days is forwarded into JFIngestGitConfig when present in endpoint_git_instance_info, and a second asserts the kwarg is omitted when absent so jf-ingest's default applies. Pushed in 08565f5.


git_configs.append(
JFIngestGitConfig(
company_slug=company_slug,
Expand All @@ -580,6 +588,7 @@ def _make_datetimes_timezone_aware(datetime_str: str):
git_strip_text_content=agent_git_config.git_strip_text_content,
check_ghc_mannequin_user_prs=agent_git_config.github_check_mannequin_users,
skip_pulling_users=skip_pulling_users,
**extra_kwargs,
)
)

Expand Down
124 changes: 123 additions & 1 deletion tests/test_config_file.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import json
import unittest
from unittest import TestCase
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch

import yaml
from jf_ingest.config import AzureDevopsAuthConfig as JFIngestAzureDevopsAuthConfig
Expand All @@ -10,9 +10,52 @@
GitConfig,
_get_git_config_from_yaml,
_get_jf_ingest_git_auth_config,
get_ingest_config,
)


def _build_ado_git_config() -> GitConfig:

Check warning on line 17 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a docstring to this function.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh1&open=AZ5Lav3liOkvN1-Ulbh1&pullRequest=457
return GitConfig(
git_url='https://ado.com',
git_provider='ado',
git_instance_slug='ado-instance-1',
git_include_projects=[],
git_exclude_projects=[],
git_include_all_repos_inside_projects=[],
git_exclude_all_repos_inside_projects=[],
git_include_repos=[],
git_exclude_repos=[],
git_include_branches={},
git_strip_text_content=False,
git_redact_names_and_urls=False,
gitlab_per_page_override=False,
git_verbose=False,
gitlab_keep_base_url=False,
creds_envvar_prefix='ORG1',
git_include_bbcloud_projects=[],
git_exclude_bbcloud_projects=[],
)


def _build_ingest_config_inputs(endpoint_git_instance_info: dict):

Check warning on line 40 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a return type hint to this function declaration.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh3&open=AZ5Lav3liOkvN1-Ulbh3&pullRequest=457

Check warning on line 40 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a docstring to this function.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh2&open=AZ5Lav3liOkvN1-Ulbh2&pullRequest=457
config = MagicMock()
config.jira_url = None
config.git_configs = [_build_ado_git_config()]
config.skip_ssl_verification = False
config.run_mode_includes_send = False
config.jira_skip_saving_data_locally = False
config.outdir = '/tmp/agent-output'

Check failure on line 47 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Make sure publicly writable directories are used safely here.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh4&open=AZ5Lav3liOkvN1-Ulbh4&pullRequest=457

Check failure

Code scanning / SonarCloud

Temporary files should not be created in publicly writable directories High test

Make sure publicly writable directories are used safely here. See more on SonarQube Cloud
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
config.jellyfish_api_base = 'https://api.jellyfish.co'

creds = MagicMock()
creds.git_instance_to_creds = {'ado-instance-1': {'ado_token': 'token-1'}}
creds.jellyfish_api_token = 'jf-token'

endpoint_git_instances_info = {'ado-instance-1': endpoint_git_instance_info}

return config, creds, endpoint_git_instances_info


class TestGitConfigGeneration(TestCase):
def test_get_git_config_from_yaml_ado_default(self):
ado_yaml_content = """
Expand Down Expand Up @@ -183,3 +226,82 @@
assert auth_config.token == 'token-2'
assert auth_config.api_version == '7.0'
assert auth_config.verify is True


class TestBackpopulationWindowDaysPassthrough(TestCase):
"""
Verifies the optional `backpopulation_window_days` value from the server endpoint
is forwarded into JFIngestGitConfig only when present, so that jf-ingest's default
applies otherwise.
"""

@patch('jf_agent.config_file_reader.IngestionConfig')
@patch('jf_agent.config_file_reader.JFIngestGitConfig')
@patch('jf_agent.config_file_reader._get_jf_ingest_git_auth_config')
@patch('jf_agent.config_file_reader.get_company_info')
def test_backpopulation_window_days_forwarded_when_present(

Check warning on line 242 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a return type hint to this function declaration.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh9&open=AZ5Lav3liOkvN1-Ulbh9&pullRequest=457
self,
mock_get_company_info,

Check warning on line 244 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh5&open=AZ5Lav3liOkvN1-Ulbh5&pullRequest=457
mock_get_auth,

Check warning on line 245 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh6&open=AZ5Lav3liOkvN1-Ulbh6&pullRequest=457
mock_jf_ingest_git_config,

Check warning on line 246 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh7&open=AZ5Lav3liOkvN1-Ulbh7&pullRequest=457
mock_ingestion_config,

Check warning on line 247 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh8&open=AZ5Lav3liOkvN1-Ulbh8&pullRequest=457
):
mock_get_company_info.return_value = {'company_slug': 'test-co'}
mock_get_auth.return_value = MagicMock()

config, creds, endpoint_git_instances_info = _build_ingest_config_inputs(
endpoint_git_instance_info={
'slug': 'ado-instance-1',
'key': 'ado-key',
'repos_dict_v2': {},
'pull_from': '2024-01-01T00:00:00',
'backpopulation_window_days': 90,
}
)

get_ingest_config(
config=config,
creds=creds,
endpoint_jira_info={},
endpoint_git_instances_info=endpoint_git_instances_info,
jf_options={},
)

self.assertEqual(mock_jf_ingest_git_config.call_count, 1)
kwargs = mock_jf_ingest_git_config.call_args.kwargs
self.assertEqual(kwargs.get('backpopulation_window_days'), 90)

@patch('jf_agent.config_file_reader.IngestionConfig')
@patch('jf_agent.config_file_reader.JFIngestGitConfig')
@patch('jf_agent.config_file_reader._get_jf_ingest_git_auth_config')
@patch('jf_agent.config_file_reader.get_company_info')
def test_backpopulation_window_days_omitted_when_absent(

Check warning on line 278 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a return type hint to this function declaration.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-UlbiC&open=AZ5Lav3liOkvN1-UlbiC&pullRequest=457
self,
mock_get_company_info,

Check warning on line 280 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh-&open=AZ5Lav3liOkvN1-Ulbh-&pullRequest=457
mock_get_auth,

Check warning on line 281 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-Ulbh_&open=AZ5Lav3liOkvN1-Ulbh_&pullRequest=457
mock_jf_ingest_git_config,

Check warning on line 282 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-UlbiA&open=AZ5Lav3liOkvN1-UlbiA&pullRequest=457
mock_ingestion_config,

Check warning on line 283 in tests/test_config_file.py

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Add a type hint to this function parameter.

See more on https://sonarcloud.io/project/issues?id=Jellyfish-AI_jf_agent&issues=AZ5Lav3liOkvN1-UlbiB&open=AZ5Lav3liOkvN1-UlbiB&pullRequest=457
):
mock_get_company_info.return_value = {'company_slug': 'test-co'}
mock_get_auth.return_value = MagicMock()

config, creds, endpoint_git_instances_info = _build_ingest_config_inputs(
endpoint_git_instance_info={
'slug': 'ado-instance-1',
'key': 'ado-key',
'repos_dict_v2': {},
'pull_from': '2024-01-01T00:00:00',
}
)

get_ingest_config(
config=config,
creds=creds,
endpoint_jira_info={},
endpoint_git_instances_info=endpoint_git_instances_info,
jf_options={},
)

self.assertEqual(mock_jf_ingest_git_config.call_count, 1)
kwargs = mock_jf_ingest_git_config.call_args.kwargs
self.assertNotIn('backpopulation_window_days', kwargs)
Loading