XML support (#2505)

* Add support for XML content-type in API

* Add management command to export to s3 the ERPs for datatourism in XML

* use freezegun to freeze time in tests

* use activities id

* Use chunks and multipart upload to avoid RAM pb

* Use distinct delay for XML files cleaning

Author: mlvernay

.isort.cfg

@@ -1,2 +1,2 @@
 [settings]
-known_third_party = Levenshtein,admin_auto_filters,autoslug,boto3,celery,colorama,corsheaders,dateutil,deepl,dj_static,django,django_better_admin_arrayfield,django_otp,django_registration,django_summernote,environ,factory,faker,frictionless,fuzz,fuzzywuzzy,geopy,ijson,import_export,magic_profanity,modeltranslation,outscraper,pandas,phonenumbers,pytest,pytest_factoryboy,requests,rest_framework,rest_framework_api_key,rest_framework_gis,reversion,schedule,scrapfly,sentry_sdk,sib_api_v3_sdk,six,splinter,stdnum,two_factor,waffle
+known_third_party = Levenshtein,admin_auto_filters,autoslug,boto3,celery,colorama,corsheaders,dateutil,deepl,dj_static,django,django_better_admin_arrayfield,django_otp,django_registration,django_summernote,environ,factory,faker,freezegun,frictionless,fuzz,fuzzywuzzy,geopy,ijson,import_export,magic_profanity,modeltranslation,outscraper,pandas,phonenumbers,pytest,pytest_factoryboy,requests,rest_framework,rest_framework_api_key,rest_framework_gis,rest_framework_xml,reversion,schedule,scrapfly,sentry_sdk,sib_api_v3_sdk,six,splinter,stdnum,two_factor,waffle

.talismanrc

@@ -3,13 +3,13 @@ fileignoreconfig:
   ignore_detectors: [filecontent]
 - filename: pnpm-lock.yaml
   ignore_detectors: [filecontent]
+- filename: .isort.cfg
+  ignore_detectors: [filecontent]
 - filename: compte/forms.py
   checksum: 028d3dedb8a51279d03b9c3c4ca6729a7e1cb3c5b96ce653fb6171517f5f238e
 - filename: templates/contrib/7-commentaire.html
   checksum: 46dbc17a97dfe7ffb619c0281c4989d02ac40a17603b696701d00132384a9dbf
 - filename: erp/export/static/schema.json
-  checksum: 2e3f20cf7ed1d425786c05f1deb3521af30d80c3e2c86f55861bab785a4fc111
-- filename: .isort.cfg
   checksum: 888e09b1fe8ead28a4561b05842cd852be1ca8a3b86264aa48469514f93cfdd2
 - filename: templates/contrib/6-accueil.html
   checksum: 6a2b50478a202eb35bf49d50ff156110eeed68287d545626753b7343bd3e3074
@@ -40,8 +40,12 @@ fileignoreconfig:
 - filename: .env.sample
   checksum: 87369459b8e37ac7fbef66697f423141bb68771f33e3f33f0a178b083aa5102b
 - filename: core/settings.py
-  checksum: 5e60668dd84288a34fc1193692c4312812318bfe5bbdb55ba5e5f42f6dc2b6d2
+  checksum: 268d050e134c6268531c274bf08ab0dbe11da7901dd2a3f9e9345112125fbf21
 - filename: pnpm-lock.yaml
   checksum: bb30cc46e656283f12141506f19deceef6f4ead92d12d23260fa94b79584ece4
 - filename: templates/erp/includes/widget_block.html
   checksum: 5fb87dbb1de0ac79bd8cd9ad167c01b67ad3bfd6fad61679188fa8e996e0576e
+- filename: erp/management/commands/export_XML_to_s3.py
+  checksum: 683b24613d882c234e8b04a674cb5ae6ee478bb877dc6bdfb223475d8cb2b6aa
+- filename: erp/management/commands/clean_S3_export_bucket.py
+  checksum: a6049e2ff31893e3ce4a9f4f661f0af25045492c91bf600c4d2485a845770b97

api/views.py

@@ -32,13 +32,14 @@
 {settings.SITE_NAME.title()} is exposing a public [API](https://en.wikipedia.org/wiki/API)
 allowing to programmatically query its database. This API embraces the
 [REST paradigm](https://en.wikipedia.org/wiki/Representational_state_transfer) as much as possible and
-exposes results in [JSON](https://en.wikipedia.org/wiki/JavaScript_Object_Notation) or [geoJSON](https://en.wikipedia.org/wiki/GeoJSON) format.
+exposes results in [JSON](https://en.wikipedia.org/wiki/JavaScript_Object_Notation), [geoJSON](https://en.wikipedia.org/wiki/GeoJSON), or [XML](https://en.wikipedia.org/wiki/XML) format.
 
 The API root entry point can be accessed at
 [`{settings.SITE_ROOT_URL}/api/`]({settings.SITE_ROOT_URL}/api/):
 - An HTML view is presented when requested through a web browser,
 - A response of type `application/json` is returned by default.
 - A response of type `application/geo+json` is returned if explicitly requested by the client and if available.
+- A response of type `application/xml` is returned if explicitly requested by the client and if available.
 ## Identification
 
 If you want to use our API, we can provide you with a key, to attach to each request to the API via the following header:

core/settings.py

@@ -188,6 +188,7 @@
     "DEFAULT_RENDERER_CLASSES": [
         "rest_framework.renderers.JSONRenderer",
         "api.renderers.GeoJSONRenderer",
+        "rest_framework_xml.renderers.XMLRenderer",
         "rest_framework.renderers.BrowsableAPIRenderer",
     ],
 }

erp/management/commands/clean_S3_export_bucket.py

@@ -11,7 +11,7 @@
 
 
 class Command(BaseCommand):
-    help = "Clean the S3 bucket storing search results exports."
+    help = "Clean the S3 bucket storing search results exports, XML exports for datatourisme."
 
     @monitor(monitor_slug="clean_S3_export_bucket")
     def handle(self, *args, **kwargs):
@@ -22,9 +22,18 @@ def handle(self, *args, **kwargs):
         files_to_delete = []
 
         for obj in response.get("Contents", []):
-            # NOTE: The presigned URLs are generated with a 24-hour validity duration; set it to 25 to avoid overlap.
-            if obj["LastModified"] < now - timedelta(hours=25):
-                files_to_delete.append({"Key": obj["Key"]})
+            key = obj["Key"]
+            last_modified = obj["LastModified"]
+            self.stdout.write(f"Checking {key} last modified {last_modified}")
+
+            if key.endswith(".xml"):
+                # NOTE: the presigned URLs are generated with a 7-day validity duration; set it to 8 to avoid overlap.
+                if last_modified < now - timedelta(days=8):
+                    files_to_delete.append({"Key": key})
+            else:
+                # NOTE: The presigned URLs are generated with a 24-hour validity duration; set it to 25 to avoid overlap.
+                if last_modified < now - timedelta(hours=25):
+                    files_to_delete.append({"Key": key})
 
         if files_to_delete:
             s3.delete_objects(Bucket=bucket_name, Delete={"Objects": files_to_delete})

erp/management/commands/export_XML_to_s3.py

@@ -0,0 +1,218 @@
+import re
+from datetime import datetime, timezone
+
+import boto3
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from rest_framework.request import Request
+from rest_framework.test import APIRequestFactory
+from rest_framework_xml.renderers import XMLRenderer
+
+from api.serializers import ErpSerializer
+from erp.models import Erp
+
+CHUNK_SIZE = 1000
+
+ACTIVITIES = [
+    255,
+    184,
+    4,
+    6,
+    1,
+    293,
+    279,
+    14,
+    257,
+    406,
+    223,
+    22,
+    243,
+    247,
+    27,
+    29,
+    253,
+    33,
+    34,
+    305,
+    38,
+    203,
+    40,
+    339,
+    41,
+    42,
+    43,
+    141,
+    298,
+    50,
+    51,
+    285,
+    450,
+    297,
+    63,
+    64,
+    375,
+    307,
+    206,
+    79,
+    62,
+    407,
+    405,
+    224,
+    85,
+    86,
+    84,
+    415,
+    95,
+    252,
+    100,
+    103,
+    109,
+    236,
+    112,
+    211,
+    411,
+    113,
+    385,
+    117,
+    209,
+    380,
+    292,
+    125,
+    432,
+    417,
+    231,
+    294,
+    310,
+    135,
+    362,
+    137,
+    145,
+    147,
+    269,
+    148,
+    152,
+    153,
+    258,
+    445,
+    413,
+    163,
+    165,
+    295,
+    171,
+    172,
+    177,
+    396,
+    220,
+    173,
+    174,
+    175,
+    176,
+    178,
+    168,
+    182,
+    183,
+    185,
+    381,
+    194,
+    371,
+    196,
+    276,
+    392,
+    414,
+    429,
+]
+
+
+class Command(BaseCommand):
+    help = "Export ERP in XML format to S3"
+
+    def handle(self, *args, **options):
+        now = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S")
+
+        server_name = settings.SITE_HOST
+        factory = APIRequestFactory(SERVER_NAME=server_name)
+        fake_request = Request(factory.get("/"))
+
+        qs = (
+            Erp.objects.published()
+            .select_related("user", "accessibilite", "activite")
+            .filter(activite__id__in=ACTIVITIES)
+            .order_by("id")
+        )
+        total = qs.count()
+        self.stdout.write(f"{total} ERPs to export...")
+
+        bucket_name = settings.S3_EXPORT_BUCKET_NAME
+        file_name = f"export_{now}_full.xml"
+
+        s3 = boto3.client(
+            "s3",
+            endpoint_url=settings.S3_EXPORT_BUCKET_ENDPOINT_URL,
+        )
+
+        # Initialize the multipart upload
+        mpu = s3.create_multipart_upload(Bucket=bucket_name, Key=file_name, ContentType="application/xml")
+        upload_id = mpu["UploadId"]
+        parts = []
+        part_number = 1
+        buffer = b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
+        renderer = XMLRenderer()
+
+        try:
+            for offset in range(0, total, CHUNK_SIZE):
+                batch = qs[offset : offset + CHUNK_SIZE]
+                self.stdout.write(f"Serialize {offset}-{offset + CHUNK_SIZE}/{total}...")
+
+                data = ErpSerializer(batch, many=True, context={"request": fake_request}).data
+                xml_str = renderer.render(data, accepted_media_type="application/xml", renderer_context={})
+                if isinstance(xml_str, str):
+                    xml_str = xml_str.encode("utf-8")
+
+                # using ErpSerializer is serializing into <root> tag, so we need to remove it, to append it to our buffer
+                inner = re.search(rb"<root>(.*)</root>", xml_str, re.DOTALL)
+                if inner:
+                    buffer += inner.group(1)
+
+                # S3 multipart  requires parts >= 5MB (except the last one)
+                if len(buffer) >= 5 * 1024 * 1024:
+                    self.stdout.write(f"Upload part {part_number} ({offset}-{offset + CHUNK_SIZE}/{total})...")
+                    response = s3.upload_part(
+                        Bucket=bucket_name,
+                        Key=file_name,
+                        PartNumber=part_number,
+                        UploadId=upload_id,
+                        Body=buffer,
+                    )
+                    parts.append({"PartNumber": part_number, "ETag": response["ETag"]})
+                    part_number += 1
+                    buffer = b""
+
+            # Remaining part
+            buffer += "</root>".encode("utf-8")
+            response = s3.upload_part(
+                Bucket=bucket_name,
+                Key=file_name,
+                PartNumber=part_number,
+                UploadId=upload_id,
+                Body=buffer,
+            )
+            parts.append({"PartNumber": part_number, "ETag": response["ETag"]})
+
+            s3.complete_multipart_upload(
+                Bucket=bucket_name,
+                Key=file_name,
+                UploadId=upload_id,
+                MultipartUpload={"Parts": parts},
+            )
+
+        except Exception as e:
+            s3.abort_multipart_upload(Bucket=bucket_name, Key=file_name, UploadId=upload_id)
+            raise e
+
+        file_url = s3.generate_presigned_url(
+            "get_object",
+            Params={"Bucket": bucket_name, "Key": file_name},
+            ExpiresIn=604800,
+        )
+
+        self.stdout.write(self.style.SUCCESS(f"Export terminated, link available during 7 days: {file_url}"))

pyproject.toml

@@ -62,6 +62,7 @@ dependencies = [
     "magic-profanity",
     "django-summernote",
     "pre-commit>=4.5.1",
+    "djangorestframework-xml>=2.0.0",
 ]
 
 [dependency-groups]
@@ -82,6 +83,7 @@ dev = [
     "legacy-cgi",
     "django-deep-translator",
     "pytest-socket",
+    "freezegun>=1.5.5",
 ]
 
 [build-system]

tests/api/tests.py

@@ -176,6 +176,13 @@ def test_list_geojson(self, api_client, initial_erp):
         )
         assert response.json() == geojson_expected_for_no_results
 
+    def test_list_xml(self, api_client, initial_erp):
+        response = api_client.get(reverse("erp-list") + "?format=xml")
+        assert response.status_code == 200
+        assert response["Content-Type"] == "application/xml; charset=utf-8"
+        content = response.content.decode("utf-8")
+        assert "<erp>" in content
+
     def test_list_can_show_drafts(self, api_client, initial_erp):
         ErpFactory(published=False)
 

tests/erp/test_export.py

@@ -14,6 +14,7 @@
 from django.contrib.gis.geos import Point
 from django.core import management
 from django.core.management import call_command
+from freezegun import freeze_time
 
 from erp.export.export import export_schema_to_csv
 from erp.export.generate_schema import generate_schema
@@ -218,7 +219,11 @@ def test_generate_schema(db, activite):
         os.remove(test_schema.name)
 
 
+CURRENT_TIME = datetime(2024, 10, 1, tzinfo=timezone.utc)
+
+
 @pytest.mark.django_db
+@freeze_time(CURRENT_TIME)
 @patch("core.mailer.BrevoMailer.send_email")
 @patch("boto3.client")
 def test_generate_csv_export(mock_boto_client, mock_send_email):
@@ -270,15 +275,9 @@ def test_generate_csv_export(mock_boto_client, mock_send_email):
     )
 
 
-CURRENT_TIME = datetime(2024, 10, 1, tzinfo=timezone.utc)
-
-
 @patch("boto3.client")
-@patch("datetime.datetime")
-def test_clean_s3_export_bucket(mock_datetime, mock_boto_client):
-    mock_datetime.now.return_value = CURRENT_TIME
-    mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
-
+@freeze_time(CURRENT_TIME)
+def test_clean_s3_export_bucket(mock_boto_client):
     mock_s3 = MagicMock()
     mock_boto_client.return_value = mock_s3