From 37fde753476f0bdf76bd75ad929b3d7a5cb5fe13 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Mon, 12 May 2025 16:08:16 +0100 Subject: [PATCH 01/29] dataload: import_to_elasticsearch: Add Country mappings Also add new fields to the es index. --- dataload/import_to_elasticsearch.py | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/dataload/import_to_elasticsearch.py b/dataload/import_to_elasticsearch.py index e64f6779..fd8f7da3 100755 --- a/dataload/import_to_elasticsearch.py +++ b/dataload/import_to_elasticsearch.py @@ -310,6 +310,15 @@ def maybe_create_index(index_name=ES_INDEX): "GNRecipientOrgCountyName": { "type": "keyword", }, + "GNBestCountryName": { + "type": "keyword", + }, + "GNBeneficiaryCountryName": { + "type": "keyword", + }, + "GNRecipientOrgCountryName": { + "type": "keyword", + } } }, # Additional funding/recipient organisation mappings @@ -447,6 +456,9 @@ def process_grant(grant, grants_file_path): # grant.additional_data.GNBestCountyName (utlanm) # grant.additional_data.GNBeneficiaryCountyName (utlanm) # grant.additional_data.GNRecipientOrgCountyName (utlanm) + # grant.additional_data.GNBestCountryName (ctrynm) + # grant.additional_data.GNRecipientCountryName (ctrynm) + # grant.additional_data.GNBeneficiaryCountryName (ctrynm) update_doc_with_other_locations(grant) # update_doc_with_undetermined needs to go last update_doc_with_undetermined(grant) @@ -637,6 +649,12 @@ def update_doc_with_other_locations(grant): except KeyError: pass + if not grant["additional_data"].get("GNBeneficiaryCountryName"): + try: + grant["additional_data"]["GNBeneficiaryCountryName"] = location["ctrynm"] + except KeyError: + pass + if not grant["additional_data"].get("GNBeneficiaryRegionName"): try: grant["additional_data"]["GNBeneficiaryRegionName"] = location["rgnnm"] @@ -652,6 +670,12 @@ def update_doc_with_other_locations(grant): # recipientOrganizationLocation if location["source"] == "recipientOrganizationLocation" or location["source"] == "recipientOrganizationPostcode": + if not grant["additional_data"].get("GNRecipientOrgCountryName"): + try: + grant["additional_data"]["GNRecipientOrgCountryName"] = location["ctrynm"] + except KeyError: + pass + if not grant["additional_data"].get("GNRecipientOrgCountyName"): try: grant["additional_data"]["GNRecipientOrgCountyName"] = location["utlanm"] @@ -687,6 +711,16 @@ def update_doc_with_other_locations(grant): except KeyError: pass + # Best Country name - Prefer beneficiary then recipient org + if not grant["additional_data"].get("GNBestCountryName"): + try: + grant["additional_data"]["GNBestCountryName"] = grant["additional_data"]["GNBeneficiaryCountryName"] + except KeyError: + try: + grant["additional_data"]["GNBestCountryName"] = grant["additional_data"]["GNRecipientOrgCountryName"] + except KeyError: + pass + # End looping over locations @@ -703,6 +737,9 @@ def update_doc_with_undetermined(grant): "GNBestCountyName", "GNRecipientOrgCountyName", "GNBeneficiaryCountyName", + "GNBeneficiaryCountryName", + "GNBestCountryName", + "GNRecipientOrgCountryName", "recipientDistrictGeoCode", "recipientDistrictName", "recipientRegionName", From fe2cee34a55f3ac5686c587e785b973e7f0320fb Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Mon, 12 May 2025 17:34:16 +0100 Subject: [PATCH 02/29] views: Add new aggregates for country filters --- grantnav/frontend/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/grantnav/frontend/views.py b/grantnav/frontend/views.py index d1c8adbb..a7fe1124 100644 --- a/grantnav/frontend/views.py +++ b/grantnav/frontend/views.py @@ -57,6 +57,10 @@ {"bool": {"should": []}}, # additional_data.GNBeneficiaryCountyName {"bool": {"should": []}}, # additional_data.GNRecipientCountyName {"bool": {"should": []}}, # additional_data.GNBestCountyName + # Country + {"bool": {"should": []}}, # additional_data.GNBestCountryName + {"bool": {"should": []}}, # additional_data.GNRecipientOrgCountryName + {"bool": {"should": []}}, # additional_data.GNBeneficiaryCountryName ] TermFacet = collections.namedtuple('TermFacet', 'field_name param_name filter_index display_name is_json facet_size') @@ -80,6 +84,9 @@ TermFacet("additional_data.GNBeneficiaryCountyName", "beneficiaryCountyName", 19, "Beneficiary County", False, 5000), TermFacet("additional_data.GNRecipientOrgCountyName", "recipientOrgCountyName", 20, "Recipient County", False, 5000), TermFacet("additional_data.GNBestCountyName", "bestCountyName", 21, "Best County", False, 5000), + TermFacet("additional_data.GNBestCountryName", "bestCountryName", 22, "Best Country", False, 5000), + TermFacet("additional_data.GNRecipientOrgCountryName", "recipientOrgCountryName", 23, "Recipieint Country", False, 5000), + TermFacet("additional_data.GNBeneficiaryCountryName", "beneficiaryCountryName", 24, "Beneficiary Country", False, 5000), ] SIZE = 20 @@ -727,6 +734,9 @@ def search(request, template_name="search.html"): filter_.append({"bool": {"should": []}}) # additional_data.GNBeneficiaryCountyName filter_.append({"bool": {"should": []}}) # additional_data.GNRecipientCountyName filter_.append({"bool": {"should": []}}) # additional_data.GNBestCountyName + filter_.append({"bool": {"should": []}}) # additional_data.GNBestCountryName + filter_.append({"bool": {"should": []}}) # additional_data.GNRecipientOrgCountryName + filter_.append({"bool": {"should": []}}) # additional_data.GNBeneficiaryCountryName json_query['aggs'] = {} for term_facet in TERM_FACETS: json_query['aggs'][term_facet.param_name] = {"terms": {"field": term_facet.field_name, From 78ca880d63c558d77250068f394eb5356e31fdd4 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Mon, 12 May 2025 19:03:00 +0100 Subject: [PATCH 03/29] frontend: filters: Add Country filter --- .../templates/components/filters/groups/geography-filters.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/grantnav/frontend/templates/components/filters/groups/geography-filters.html b/grantnav/frontend/templates/components/filters/groups/geography-filters.html index 6e73d17c..893381a1 100644 --- a/grantnav/frontend/templates/components/filters/groups/geography-filters.html +++ b/grantnav/frontend/templates/components/filters/groups/geography-filters.html @@ -26,18 +26,21 @@ {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=True aggregate_data=results.aggregations.recipientRegionName filter_title="Country and Region" %} {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=False finder=True finder_class="best_county" aggregate_data=results.aggregations.bestCountyName filter_title="County" %} {% include 'components/filters/lists/filter-list-district.html' with open=True %} + {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=True aggregate_data=results.aggregations.bestCountryName filter_title="Country" filter_id="bestCountryName" %} From a1dc5dd4fd68cddece649c13180147b3e79a0bdd Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Mon, 12 May 2025 19:03:00 +0100 Subject: [PATCH 04/29] frontend: filters: Add Country filter --- .../templates/components/filters/groups/geography-filters.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/grantnav/frontend/templates/components/filters/groups/geography-filters.html b/grantnav/frontend/templates/components/filters/groups/geography-filters.html index 893381a1..c9112490 100644 --- a/grantnav/frontend/templates/components/filters/groups/geography-filters.html +++ b/grantnav/frontend/templates/components/filters/groups/geography-filters.html @@ -23,6 +23,7 @@
+ {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=True aggregate_data=results.aggregations.bestCountryName filter_title="Country" filter_id="bestCountryName" %} {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=True aggregate_data=results.aggregations.recipientRegionName filter_title="Country and Region" %} {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=False finder=True finder_class="best_county" aggregate_data=results.aggregations.bestCountyName filter_title="County" %} {% include 'components/filters/lists/filter-list-district.html' with open=True %} @@ -30,6 +31,7 @@
From 863243c50d90d0564033e4d8fd856ac9a60f6739 Mon Sep 17 00:00:00 2001 From: Michael Wood Date: Thu, 19 Jun 2025 11:18:50 +0100 Subject: [PATCH 08/29] wip --- .../templates/components/filters/groups/geography-filters.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/grantnav/frontend/templates/components/filters/groups/geography-filters.html b/grantnav/frontend/templates/components/filters/groups/geography-filters.html index 4ef74a80..6c64dc7c 100644 --- a/grantnav/frontend/templates/components/filters/groups/geography-filters.html +++ b/grantnav/frontend/templates/components/filters/groups/geography-filters.html @@ -27,7 +27,6 @@ {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=True aggregate_data=results.aggregations.recipientRegionName filter_title="Country and Region" %} {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=False finder=True finder_class="best_county" aggregate_data=results.aggregations.bestCountyName filter_title="County" %} {% include 'components/filters/lists/filter-list-district.html' with open=True %} - {% include 'components/filters/lists/filter-list-generic-checkboxes.html' with open=True aggregate_data=results.aggregations.bestCountryName filter_title="Country" filter_id="bestCountryName" %}