validate program checs for individuals too

pyproject.toml

@@ -97,32 +97,6 @@ dev-dependencies = [
     "ruff>=0.9.3",
 ]
 
-[tool.black]
-line-length = 120
-include = '\.pyi?$'
-exclude = '''
-/(
-    \.git
-  | \.pytest_cache
-  | \.tox
-  | \.venv
-  | ~build
-  | build
-  | ops
-  | migrations
-)/
-'''
-
-#[tool.isort]
-#profile = "black"
-#line_length = 120
-#default_section = "THIRDPARTY"
-#known_first_party = []
-#known_django = "django"
-#sections = ["FUTURE","STDLIB","DJANGO","THIRDPARTY","FIRSTPARTY","LOCALFOLDER"]
-#include_trailing_comma = true
-#skip = ["migrations", "snapshots", ".venv"]
-
 
 [tool.django-stubs]
 django_settings_module = "country_workspace.config.settings"

ruff.toml

@@ -80,8 +80,7 @@ force-wrap-aliases = true
 combine-as-imports = true
 extra-standard-library = ["path"]
 known-third-party = ['django']
-known-first-party = ['hope_*']
-known-local-folder = ["country_workspace"]
+known-first-party = ['country_workspace']
 no-sections = false
 lines-after-imports = 1
 relative-imports-order = "furthest-to-closest"

src/country_workspace/apps.py

@@ -15,6 +15,7 @@ def ready(self) -> None:
         import django_celery_boost.admin
 
         import country_workspace.compat.admin_extra_buttons as c
+
         from .utils import flags  # noqa
 
         admin_extra_buttons.api.confirm_action = c.confirm_action

src/country_workspace/cache/manager.py

@@ -10,10 +10,11 @@
 from redis_lock.django_cache import RedisCache
 from sentry_sdk import capture_exception
 
-from .signals import cache_get, cache_invalidate, cache_set
 from country_workspace import VERSION
 from country_workspace.state import state
 
+from .signals import cache_get, cache_invalidate, cache_set
+
 if TYPE_CHECKING:
     from ..models import Office, Program
 logger = logging.getLogger(__name__)

src/country_workspace/contrib/hope/client.py

@@ -8,9 +8,10 @@
 from constance import config
 from requests.exceptions import RequestException
 
-from .signals import hope_request_end, hope_request_start
 from country_workspace.exceptions import RemoteError
 
+from .signals import hope_request_end, hope_request_start
+
 if TYPE_CHECKING:
     JsonType = None | int | str | bool | list["JsonType"] | dict[str, "JsonType"]
     FlatJsonType = dict[str, str | int | bool]

src/country_workspace/contrib/hope/geo.py

@@ -4,12 +4,12 @@
 from django import forms
 from django.core.exceptions import ValidationError
 from django.utils.text import slugify
-
 from hope_flex_fields.mixin import ChildFieldMixin
 
+from country_workspace.cache.manager import cache_manager
+
 from ...exceptions import RemoteError
 from .client import HopeClient
-from country_workspace.cache.manager import cache_manager
 
 logger = logging.getLogger(__name__)
 

src/country_workspace/contrib/hope/remotes/country.py

@@ -1,7 +1,6 @@
 from typing import TYPE_CHECKING
 
 from django import forms
-
 from hope_flex_fields.attributes.abstract import AbstractAttributeHandler, AttributeHandlerConfig
 
 if TYPE_CHECKING:

src/country_workspace/contrib/hope/sync/office.py

@@ -1,12 +1,12 @@
 from io import TextIOBase
 
 from django.core.cache import cache
-
 from hope_flex_fields.models import DataChecker
 
+from country_workspace.models import Office, Program, SyncLog
+
 from .. import constants
 from ..client import HopeClient
-from country_workspace.models import Office, Program, SyncLog
 
 
 def sync_offices(stdout: TextIOBase | None = None) -> dict[str, int]:

src/country_workspace/datasources/rdi.py

@@ -1,7 +1,6 @@
 import io
 
 from django.db.transaction import atomic
-
 from hope_smart_import.readers import open_xls_multi
 
 from country_workspace.models import AsyncJob, Batch, Household

src/country_workspace/models/base.py

@@ -14,7 +14,6 @@
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
-
     from hope_flex_fields.models import DataChecker
 
     from country_workspace.models import Office, Program

src/country_workspace/models/household.py

@@ -3,12 +3,12 @@
 
 import reversion
 from django.db import models
+from django.utils import timezone
 
 from .base import BaseModel, Validable
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
-
     from hope_flex_fields.models import DataChecker
 
     from .individual import Individual
@@ -37,11 +37,19 @@ def country_office(self) -> "Office":
         return self.batch.program.country_office
 
     def validate_with_checker(self) -> bool:
-        super().validate_with_checker()
-        errors = self.program.beneficiary_validator.validate(self)
-        if errors:
-            self.errors["dct"] = errors
-        self.save(update_fields=["errors"])
+        hh_valid = True
+        for ind in self.members.all():
+            if not ind.validate_with_checker():
+                hh_valid = False
+        if hh_valid:
+            super().validate_with_checker()
+            errors = self.program.beneficiary_validator.validate(self)
+            if errors:
+                self.errors["dct"] = errors
+        else:
+            self.errors["dct"] = ["Some member did not validate"]
+        self.last_checked = timezone.now()
+        self.save(update_fields=["errors", "last_checked"])
         return not bool(self.errors)
 
     # Business methods

src/country_workspace/models/individual.py

@@ -22,8 +22,8 @@ def checker(self) -> "DataChecker":
 
     @cached_property
     def program(self) -> "DataChecker":
-        return self.household.batch.program
+        return self.batch.program
 
     @cached_property
     def country_office(self) -> "DataChecker":
-        return self.household.batch.program.country_office
+        return self.batch.program.country_office

src/country_workspace/models/program.py

@@ -2,14 +2,14 @@
 
 from django.db import models
 from django.utils.translation import gettext as _
+from hope_flex_fields.models import DataChecker
 from strategy_field.fields import StrategyField
 from strategy_field.utils import fqn
 
-from hope_flex_fields.models import DataChecker
+from country_workspace.models.office import Office
 
 from ..validators.registry import NoopValidator, beneficiary_validator_registry
 from .base import BaseModel, Validable
-from country_workspace.models.office import Office
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet

src/country_workspace/tasks.py

@@ -39,7 +39,7 @@ def sync_job_task(pk: int, version: int) -> dict[str, Any]:
             pk=pk,
             version=version,
         )
-    except AsyncJob.DoesNotExist as e:
+    except AsyncJob.DoesNotExist as e:  # pragma: no cover
         sentry_sdk.capture_exception(e)
         raise e
 

src/country_workspace/versioning/checkers.py

@@ -1,5 +1,4 @@
 from django import forms
-
 from hope_flex_fields.models import DataChecker, FieldDefinition, Fieldset
 
 from country_workspace.contrib.hope.constants import HOUSEHOLD_CHECKER_NAME, INDIVIDUAL_CHECKER_NAME

src/country_workspace/versioning/hope_fields.py

@@ -1,7 +1,6 @@
 from django import forms
 from django.conf import settings
 from django.utils.text import slugify
-
 from hope_flex_fields.models import FieldDefinition
 
 from country_workspace.models import SyncLog

src/country_workspace/versioning/scripts/0001_initial.py

@@ -1,6 +1,5 @@
 # Generated by Django 5.1.1 on 2024-10-09 06:19
 from flags.state import enable_flag
-
 from hope_flex_fields.models import DataChecker, Fieldset
 
 from country_workspace.contrib.hope.constants import HOUSEHOLD_CHECKER_NAME, INDIVIDUAL_CHECKER_NAME

src/country_workspace/workspaces/admin/cleaners/actions.py

@@ -6,14 +6,15 @@
 from django.utils.translation import gettext as _
 from strategy_field.utils import fqn
 
+from country_workspace.models import AsyncJob
+from country_workspace.state import state
+from country_workspace.workspaces.admin.forms import BulkUpdateExportForm
+
 from .bulk_update import bulk_update_export_template
 from .calculate_checksum import calculate_checksum_impl
 from .mass_update import MassUpdateForm, mass_update_impl
 from .regex import RegexUpdateForm, regex_update_impl
 from .validate import validate_queryset
-from country_workspace.models import AsyncJob
-from country_workspace.state import state
-from country_workspace.workspaces.admin.forms import BulkUpdateExportForm
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet

src/country_workspace/workspaces/admin/cleaners/bulk_update.py

@@ -6,11 +6,10 @@
 from django import forms
 from django.apps import apps
 from django.core.exceptions import ObjectDoesNotExist
-from xlsxwriter import Workbook
-
 from hope_flex_fields.models import DataChecker, FlexField
 from hope_flex_fields.xlsx import get_format_for_field
 from hope_smart_import.readers import open_xls
+from xlsxwriter import Workbook
 
 from country_workspace.models import AsyncJob, Program
 from country_workspace.storages import MEDIA_STORAGE

src/country_workspace/workspaces/admin/cleaners/mass_update.py

@@ -4,15 +4,13 @@
 from django.db import transaction
 from django.forms import MultiValueField, widgets
 from django.utils.text import slugify
-from strategy_field.utils import fqn
-
 from hope_flex_fields.fields import FlexFormMixin
+from strategy_field.utils import fqn
 
 from .base import BaseActionForm
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
-
     from hope_flex_fields.models import DataChecker
 
     from country_workspace.types import Beneficiary

src/country_workspace/workspaces/admin/cleaners/regex.py

@@ -4,12 +4,12 @@
 from django import forms
 from django.db import transaction
 
-from .base import BaseActionForm
 from country_workspace.utils.flex_fields import get_checker_fields
 
+from .base import BaseActionForm
+
 if TYPE_CHECKING:
     from django.db.models import QuerySet
-
     from hope_flex_fields.models import DataChecker
 
     from country_workspace.types import Beneficiary

src/country_workspace/workspaces/admin/cleaners/validate.py

@@ -1,20 +1,44 @@
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from django.db.models import Model, QuerySet
 
+from country_workspace.models import AsyncJob, Household, Program
+
+if TYPE_CHECKING:
+    from country_workspace.models.base import Validable
+
 logger = logging.getLogger(__name__)
 
 
 def validate_queryset(queryset: QuerySet[Model], **kwargs: Any) -> dict[str, int]:
     valid = invalid = num = 0
+    entry: "Validable"
     try:
         for __, entry in enumerate(queryset, 1):
             if entry.validate_with_checker():
                 valid += 1
             else:
                 invalid += 1
-    except Exception as e:
+    except Exception as e:  # pragma: no cover
+        logger.exception(e)
+        raise
+
+    return {"valid": valid, "invalid": invalid, "total": num}
+
+
+def validate_program(job: AsyncJob) -> dict[str, int]:
+    valid = invalid = num = 0
+    hh: Household
+    try:
+        p: Program = job.program
+        for hh in Household.objects.filter(batch__program=p):
+            if hh.validate_with_checker():
+                valid += 1
+            else:
+                invalid += 1
+    except Exception as e:  # pragma: no cover
         logger.exception(e)
+        raise
 
     return {"valid": valid, "invalid": invalid, "total": num}

src/country_workspace/workspaces/admin/hh_ind.py

@@ -18,7 +18,7 @@
 from ...state import state
 from ..options import WorkspaceModelAdmin
 from .cleaners import actions
-from .cleaners.validate import validate_queryset
+from .cleaners.validate import validate_program
 
 if TYPE_CHECKING:
     from hope_flex_fields.forms import FlexForm
@@ -102,15 +102,14 @@ def validate_program(self, request: HttpRequest) -> "HttpResponse":
         opts = self.model._meta
         job = AsyncJob.objects.create(
             description="Validate Program %s" % opts.proxy_for_model._meta.verbose_name_plural,
-            type=AsyncJob.JobType.ACTION,
+            type=AsyncJob.JobType.TASK,
             owner=state.request.user,
-            action=fqn(validate_queryset),
+            action=fqn(validate_program),
             program=state.program,
-            config={"pks": "__all__", "model_name": opts.label},
+            config={},
         )
         job.queue()
         self.message_user(request, "Task scheduled", messages.SUCCESS)
-        return job
 
     @button()
     def view_raw_data(self, request: HttpRequest, pk: str) -> "HttpResponse":

src/country_workspace/workspaces/admin/program.py

@@ -13,6 +13,10 @@
 from django.utils.translation import gettext as _
 from strategy_field.utils import fqn
 
+from country_workspace.constants import BATCH_NAME_DEFAULT
+from country_workspace.contrib.aurora.pipeline import import_from_aurora
+from country_workspace.state import state
+
 from ...contrib.aurora.forms import ImportAuroraForm
 from ...contrib.kobo.forms import ImportKoboForm
 from ...contrib.kobo.sync import import_data as import_from_kobo
@@ -24,9 +28,6 @@
 from ..sites import workspace
 from .cleaners.bulk_update import bulk_update_household, bulk_update_individual
 from .forms import BulkUpdateImportForm, ImportFileForm
-from country_workspace.constants import BATCH_NAME_DEFAULT
-from country_workspace.contrib.aurora.pipeline import import_from_aurora
-from country_workspace.state import state
 
 if TYPE_CHECKING:
     from hope_flex_fields.models import DataChecker

src/country_workspace/workspaces/models.py

@@ -3,7 +3,6 @@
 import reversion
 from django.db import models
 from django.utils.functional import cached_property
-
 from hope_flex_fields.models import DataChecker
 
 from country_workspace.models import AsyncJob, Batch, Household, Individual, Office, Program