Skip to content

Commit 4194d43

Browse files
g-pechoring-pechorin
committed
fiox
1 parent 3f61c63 commit 4194d43

74 files changed

Lines changed: 339 additions & 362 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ jobs:
1111
runs-on: ubuntu-latest
1212
steps:
1313
- name: Check out the repo
14-
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
14+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
1515
with:
1616
submodules: "recursive"
1717
ref: ${{ github.ref }}

‎app/airflow/dags/SR_processing_dag.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
from datetime import datetime, timedelta
2+
23
from airflow import DAG
34
from airflow.operators.empty import EmptyOperator
4-
from libs.utils import create_task, validate_params_SR_processing
5+
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT, AIRFLOW_DEBUG_MODE
56
from libs.SR_processing.core import (
6-
process_data_dictionary,
77
process_and_create_scan_report_entries,
8+
process_data_dictionary,
89
)
9-
from libs.utils import connect_to_storage
10-
from libs.settings import AIRFLOW_DEBUG_MODE, AIRFLOW_DAGRUN_TIMEOUT
10+
from libs.utils import connect_to_storage, create_task, validate_params_SR_processing
1111

1212
"""
1313
This DAG automates the process of creating scan report tables, fields and values

‎app/airflow/dags/auto_mapping_dag.py‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,35 @@
11
from datetime import datetime, timedelta
2+
23
from airflow import DAG
34
from airflow.operators.empty import EmptyOperator
4-
from libs.auto_mapping.find_R_concepts_to_reuse import (
5-
find_matching_field,
6-
find_matching_value,
7-
create_reusing_concepts,
8-
delete_R_concepts,
9-
)
10-
11-
from libs.auto_mapping.find_standard_V_concepts import (
12-
find_standard_concepts,
13-
create_standard_concepts,
14-
)
15-
165
from libs.auto_mapping.core_get_existing_concepts import (
176
delete_mapping_rules,
187
find_existing_concepts,
198
)
209
from libs.auto_mapping.core_prep_rules_creation import (
21-
find_dest_table_and_person_field_id,
22-
find_date_fields,
23-
find_concept_fields,
2410
find_additional_fields,
11+
find_concept_fields,
12+
find_date_fields,
13+
find_dest_table_and_person_field_id,
2514
)
2615
from libs.auto_mapping.core_rules_creation import create_mapping_rules
16+
from libs.auto_mapping.find_R_concepts_to_reuse import (
17+
create_reusing_concepts,
18+
delete_R_concepts,
19+
find_matching_field,
20+
find_matching_value,
21+
)
22+
from libs.auto_mapping.find_standard_V_concepts import (
23+
create_standard_concepts,
24+
find_standard_concepts,
25+
)
2726
from libs.auto_mapping.search_recommendations import process_search_recommendations
27+
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT, AIRFLOW_DEBUG_MODE, SEARCH_ENABLED
2828
from libs.utils import (
2929
create_task,
30-
validate_params_auto_mapping,
3130
update_job_status_on_failure,
31+
validate_params_auto_mapping,
3232
)
33-
from libs.settings import AIRFLOW_DEBUG_MODE, SEARCH_ENABLED, AIRFLOW_DAGRUN_TIMEOUT
3433

3534
"""
3635
This DAG automates the process of creating and reusing concepts from scan reports and generating mapping rules.

‎app/airflow/dags/libs/SR_processing/core.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,28 @@
1+
import csv
12
import logging
23
import os
3-
from openpyxl import load_workbook
4-
from libs.utils import pull_validated_params
5-
import csv
64
from io import StringIO
5+
from typing import List, Tuple
6+
7+
from airflow.providers.postgres.hooks.postgres import PostgresHook
8+
from libs.enums import JobStageType, StageStatusType
9+
from libs.queries import create_temp_data_dictionary_table_query, create_values_query
10+
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT
711
from libs.SR_processing.db_services import (
812
create_field_entries,
9-
update_temp_data_dictionary_table,
1013
create_temp_field_values_table,
1114
delete_temp_tables,
15+
update_temp_data_dictionary_table,
1216
)
1317
from libs.SR_processing.helpers import (
14-
remove_BOM,
15-
process_four_item_dict,
1618
get_unique_table_names,
19+
process_four_item_dict,
20+
remove_BOM,
1721
transform_scan_report_sheet_table,
1822
)
1923
from libs.storage_services import download_blob_to_tmp
20-
from airflow.providers.postgres.hooks.postgres import PostgresHook
21-
from typing import List, Tuple
22-
from libs.queries import create_values_query, create_temp_data_dictionary_table_query
23-
from libs.enums import JobStageType, StageStatusType
24-
from libs.utils import update_job_status
25-
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT
24+
from libs.utils import pull_validated_params, update_job_status
25+
from openpyxl import load_workbook
2626

2727
# PostgreSQL connection hook
2828
pg_hook = PostgresHook(

‎app/airflow/dags/libs/SR_processing/db_services.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
1-
from typing import List, Any, Dict, Tuple
2-
from openpyxl.worksheet.worksheet import Worksheet
3-
from libs.queries import create_fields_query
41
import logging
52
from collections import defaultdict
3+
from typing import Any, Dict, List, Tuple
4+
65
from airflow.providers.postgres.hooks.postgres import PostgresHook
76
from libs.enums import JobStageType, StageStatusType
7+
from libs.queries import create_fields_query
8+
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT, AIRFLOW_DEBUG_MODE
89
from libs.utils import update_job_status
9-
from libs.settings import AIRFLOW_DEBUG_MODE, AIRFLOW_DAGRUN_TIMEOUT
10+
from openpyxl.worksheet.worksheet import Worksheet
1011

1112
# PostgreSQL connection hook
1213
pg_hook = PostgresHook(

‎app/airflow/dags/libs/SR_processing/helpers.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
from typing import List, Any, Dict
2-
from openpyxl.worksheet.worksheet import Worksheet
31
import logging
42
from collections import defaultdict
3+
from typing import Any, Dict, List
4+
5+
from openpyxl.worksheet.worksheet import Worksheet
56

67

78
def get_unique_table_names(worksheet: Worksheet) -> List[str]:

‎app/airflow/dags/libs/auto_mapping/core_get_existing_concepts.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
1-
from libs.utils import (
2-
update_job_status,
3-
JobStageType,
4-
StageStatusType,
5-
pull_validated_params,
6-
)
7-
from airflow.providers.postgres.hooks.postgres import PostgresHook
81
import logging
2+
3+
from airflow.providers.postgres.hooks.postgres import PostgresHook
94
from libs.queries import (
105
create_existing_concepts_table_query,
116
find_existing_concepts_query,
127
find_source_field_id_query,
138
)
149
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT
10+
from libs.utils import (
11+
JobStageType,
12+
StageStatusType,
13+
pull_validated_params,
14+
update_job_status,
15+
)
1516

1617
# PostgreSQL connection hook
1718
pg_hook = PostgresHook(

‎app/airflow/dags/libs/auto_mapping/core_prep_rules_creation.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
1-
from libs.utils import (
2-
update_job_status,
3-
JobStageType,
4-
StageStatusType,
5-
pull_validated_params,
6-
)
7-
from airflow.providers.postgres.hooks.postgres import PostgresHook
81
import logging
2+
3+
from airflow.providers.postgres.hooks.postgres import PostgresHook
94
from libs.queries import (
10-
find_dest_table_and_person_field_id_query,
11-
find_dates_fields_query,
125
find_concept_fields_query,
6+
find_dates_fields_query,
7+
find_dest_table_and_person_field_id_query,
138
)
149
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT
10+
from libs.utils import (
11+
JobStageType,
12+
StageStatusType,
13+
pull_validated_params,
14+
update_job_status,
15+
)
1516

1617
# PostgreSQL connection hook
1718
pg_hook = PostgresHook(

‎app/airflow/dags/libs/auto_mapping/core_rules_creation.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
from airflow.providers.postgres.hooks.postgres import PostgresHook
21
import logging
2+
33
from airflow.exceptions import AirflowException
4+
from airflow.providers.postgres.hooks.postgres import PostgresHook
5+
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT, AIRFLOW_DEBUG_MODE
46
from libs.utils import (
5-
update_job_status,
67
JobStageType,
78
StageStatusType,
89
pull_validated_params,
10+
update_job_status,
911
)
10-
from libs.settings import AIRFLOW_DEBUG_MODE, AIRFLOW_DAGRUN_TIMEOUT
11-
1212

1313
# PostgreSQL connection hook
1414
pg_hook = PostgresHook(

‎app/airflow/dags/libs/auto_mapping/find_R_concepts_to_reuse.py‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
1-
from libs.utils import (
2-
update_job_status,
3-
JobStageType,
4-
StageStatusType,
5-
pull_validated_params,
6-
)
7-
from airflow.providers.postgres.hooks.postgres import PostgresHook
81
import logging
2+
3+
from airflow.providers.postgres.hooks.postgres import PostgresHook
94
from libs.queries import (
5+
create_reuse_concept_query,
106
create_temp_reuse_table_query,
117
find_m_concepts_query,
128
find_m_concepts_query_field_level,
139
find_object_id_query,
1410
validate_reused_concepts_query,
15-
create_reuse_concept_query,
1611
)
1712
from libs.settings import AIRFLOW_DAGRUN_TIMEOUT
13+
from libs.utils import (
14+
JobStageType,
15+
StageStatusType,
16+
pull_validated_params,
17+
update_job_status,
18+
)
1819

1920
# PostgreSQL connection hook
2021
pg_hook = PostgresHook(

0 commit comments

Comments
 (0)