Merge pull request #169 from unicef/staging_deployment

Staging Deployment

Author: vikashkum05

proco/data_sources/tasks.py

@@ -704,20 +704,21 @@ def cleanup_school_master_rows():
     if task_instance:
         logger.debug('Not found running job for school master cleanup task: {}'.format(task_key))
         # Delete all the old records where more than 1 record are in DRAFT/UPDATED_IN_DRAFT or
-        # ROW_STATUS_DRAFT_LOCKED/ROW_STATUS_UPDATED_IN_DRAFT_LOCKED for same School GIGA ID
+        # ROW_STATUS_DRAFT_LOCKED/ROW_STATUS_UPDATED_IN_DRAFT_LOCKED/ROW_STATUS_DELETED for same School GIGA ID
         rows_with_more_than_1_record_in_draft = sources_models.SchoolMasterData.objects.filter(
             status__in=[
                 sources_models.SchoolMasterData.ROW_STATUS_DRAFT,
                 sources_models.SchoolMasterData.ROW_STATUS_UPDATED_IN_DRAFT,
                 sources_models.SchoolMasterData.ROW_STATUS_DRAFT_LOCKED,
                 sources_models.SchoolMasterData.ROW_STATUS_UPDATED_IN_DRAFT_LOCKED,
+                sources_models.SchoolMasterData.ROW_STATUS_DELETED,
             ]
         ).values('school_id_giga', 'country_id').annotate(
             total_records=Count('school_id_giga', distinct=False),
         ).order_by('-total_records', 'school_id_giga', 'country_id').filter(total_records__gt=1)
 
         logger.debug('Queryset to get all the old records to delete where more than 1 record are in DRAFT/'
-                     'UPDATED_IN_DRAFT/ROW_STATUS_DRAFT_LOCKED/ROW_STATUS_UPDATED_IN_DRAFT_LOCKED '
+                     'UPDATED_IN_DRAFT/ROW_STATUS_DRAFT_LOCKED/ROW_STATUS_UPDATED_IN_DRAFT_LOCKED/ROW_STATUS_DELETED '
                      'for same School GIGA ID: {0}'.format(rows_with_more_than_1_record_in_draft.query))
 
         for row in rows_with_more_than_1_record_in_draft:
@@ -727,26 +728,48 @@ def cleanup_school_master_rows():
             ).order_by('-created')[1:]:
                 row_to_delete.delete()
         task_instance.info('Deleted rows where more than 1 record are in DRAFT/'
-                           'UPDATED_IN_DRAFT/ROW_STATUS_DRAFT_LOCKED/ROW_STATUS_UPDATED_IN_DRAFT_LOCKED '
+                           'UPDATED_IN_DRAFT/ROW_STATUS_DRAFT_LOCKED/ROW_STATUS_UPDATED_IN_DRAFT_LOCKED/ROW_STATUS_DELETED '
                            'for same School GIGA ID')
 
-        # Delete all the old records where more than 1 record are in is_read=True for same School GIGA ID
+        # At least keep 1 PUBLISHED row for each school in the school master table
+        rows_with_more_than_1_record_in_published = sources_models.SchoolMasterData.objects.filter(
+            status=sources_models.SchoolMasterData.ROW_STATUS_PUBLISHED,
+        ).values('school_id_giga', 'country_id').annotate(
+            total_records=Count('school_id_giga', distinct=False),
+        ).order_by('-total_records').filter(total_records__gt=1)
+
+        logger.debug('Queryset to get all the old records to delete where more than 1 record are in PUBLISHED '
+                     'for same School GIGA ID: {0}'.format(rows_with_more_than_1_record_in_published.query))
+
+        for row in rows_with_more_than_1_record_in_published:
+            for row_to_delete in sources_models.SchoolMasterData.objects.filter(
+                school_id_giga=row['school_id_giga'],
+                country_id=row['country_id'],
+            ).order_by('-published_at')[1:]:
+                row_to_delete.delete()
+        task_instance.info('Deleted rows where more than 1 record are in PUBLISHED for same School GIGA ID')
+
+        # Delete all the old records where more than 1 record are in is_read=True
+        # by keeping at least 1 PUBLISHED row for same School GIGA ID
         rows_with_more_than_1_record_in_read = sources_models.SchoolMasterData.objects.filter(
             is_read=True,
         ).values('school_id_giga', 'country_id').annotate(
             total_records=Count('school_id_giga', distinct=False),
         ).order_by('-total_records').filter(total_records__gt=1)
 
-        logger.debug('Queryset to get all the old records to delete where more than 1 record are in is_read=True '
-                     'for same School GIGA ID: {0}'.format(rows_with_more_than_1_record_in_read.query))
+        logger.debug('Queryset to get all the old records to delete where more than 1 record are in is_read=True'
+                     ' by keeping at least 1 PUBLISHED row for same School GIGA ID: {0}'.format(
+            rows_with_more_than_1_record_in_read.query))
 
         for row in rows_with_more_than_1_record_in_read:
             for row_to_delete in sources_models.SchoolMasterData.objects.filter(
                 school_id_giga=row['school_id_giga'],
                 country_id=row['country_id'],
             ).order_by('-published_at')[1:]:
-                row_to_delete.delete()
-        task_instance.info('Deleted rows where more than 1 record are in is_read=True for same School GIGA ID')
+                if row_to_delete.status != sources_models.SchoolMasterData.ROW_STATUS_PUBLISHED:
+                    row_to_delete.delete()
+        task_instance.info('Deleted rows where more than 1 record are in is_read=True '
+                           'by keeping at least 1 PUBLISHED row for same School GIGA ID')
 
         background_task_utilities.task_on_complete(task_instance)
     else:

proco/data_sources/utils.py

@@ -2,7 +2,7 @@
 import logging
 import os
 from datetime import timedelta
-from typing import Optional
+from typing import Optional, Any
 
 import delta_sharing
 import numpy as np
@@ -100,41 +100,54 @@ def normalize_qos_data_frame(df):
     return df
 
 
-def has_changes_for_review(row, school):
+def _values_equal(a: Any, b: Any) -> bool:
+    """
+    Null-safe equality check.
+    Treats NaN, NaT, and None as equal if both sides are missing.
+    """
+    if (pd.isna(a) and pd.isna(b)) or (a is None and b is None):
+        return True
+    return a == b
+
+def has_changes_for_review(row, school) -> bool:
+    """
+    Compare a DataFrame row with an existing School model instance.
+    Returns True if any meaningful change is detected; False otherwise.
+
+    This preserves the original control flow and column accesses, but uses
+    null-safe comparisons so NaN/NaT/None do not cause false positives.
+    """
     if school:
-        if row['school_name'].lower() != school.name.lower():
+        if not _values_equal(row['school_name'].lower(), school.name.lower()):
             return True
 
         old_external_id = None \
             if core_utilities.is_blank_string(school.external_id) else str(school.external_id).lower()
         new_external_id = None \
             if core_utilities.is_blank_string(row['school_id_govt']) else str(row['school_id_govt']).lower()
-
-        if old_external_id != new_external_id:
+        if not _values_equal(old_external_id, new_external_id):
             return True
 
         old_admin1_id = None
         if school.admin1:
             old_admin1_id = str(school.admin1.giga_id_admin).lower()
         new_admin1_id = None \
             if core_utilities.is_blank_string(row['admin1_id_giga']) else str(row['admin1_id_giga']).lower()
-
-        if old_admin1_id != new_admin1_id:
+        if not _values_equal(old_admin1_id, new_admin1_id):
             return True
 
         old_admin2_id = None
         if school.admin2:
             old_admin2_id = str(school.admin2.giga_id_admin).lower()
         new_admin2_id = None \
             if core_utilities.is_blank_string(row['admin2_id_giga']) else str(row['admin2_id_giga']).lower()
-
-        if old_admin2_id != new_admin2_id:
+        if not _values_equal(old_admin2_id, new_admin2_id):
             return True
 
         old_lat = school.geopoint.y
         new_lat = row['latitude']
         if (
-            old_lat != new_lat and
+            not _values_equal(old_lat, new_lat) and
             (
                 str(old_lat).split('.')[0] != str(new_lat).split('.')[0] or
                 str(old_lat).split('.')[1][:5] != str(new_lat).split('.')[1][:5]
@@ -145,7 +158,7 @@ def has_changes_for_review(row, school):
         old_long = school.geopoint.x
         new_long = row['longitude']
         if (
-            old_long != new_long and
+            not _values_equal(old_long, new_long) and
             (
                 str(old_long).split('.')[0] != str(new_long).split('.')[0] or
                 str(old_long).split('.')[1][:5] != str(new_long).split('.')[1][:5]
@@ -157,8 +170,7 @@ def has_changes_for_review(row, school):
             if core_utilities.is_blank_string(school.education_level) else str(school.education_level).lower()
         new_education_level = None \
             if core_utilities.is_blank_string(row['education_level']) else str(row['education_level']).lower()
-
-        if old_education_level != new_education_level:
+        if not _values_equal(old_education_level, new_education_level):
             return True
 
         school_rt_instance = SchoolRealTimeRegistration.objects.filter(school=school).order_by('-created').first()
@@ -171,7 +183,7 @@ def has_changes_for_review(row, school):
         ):
             new_connectivity_rt = str(row['connectivity_RT']).lower() in core_configs.true_choices
 
-        if old_connectivity_rt != new_connectivity_rt:
+        if not _values_equal(old_connectivity_rt, new_connectivity_rt):
             return True
         return False
     return True