feat: migrate to ResilientLogger

Switch using the resilient logger's audit logging over the project's own
audit log.

The old model can be removed later when confirmed that the logging is
functioning correctly.

Refs: ATV-254

Author: nicobav

README.md

@@ -117,6 +117,12 @@ run all the formatting tools as git hooks automatically before a
 commit.
 
 
+## Audit logging
+
+Audit logging is implemented using the [resilient-logger](https://github.com/City-of-Helsinki/resilient-logger) library.
+See audit_log package.
+
+
 ## Running tests
 
 * Set the `DEBUG` environment variable to `1`.

atv/settings.py

@@ -21,6 +21,12 @@
     default_var_root = environ.Path(checkout_dir("var"))
 
 env = environ.Env(
+    # Resilient logger config
+    AUDIT_LOG_ENVIRONMENT=(str, ""),
+    AUDIT_LOG_ES_URL=(str, ""),
+    AUDIT_LOG_ES_INDEX=(str, ""),
+    AUDIT_LOG_ES_USERNAME=(str, ""),
+    AUDIT_LOG_ES_PASSWORD=(str, ""),
     DEBUG=(bool, False),
     SECRET_KEY=(str, ""),
     MEDIA_ROOT=(environ.Path(), default_var_root("media")),
@@ -51,6 +57,7 @@
     FIELD_ENCRYPTION_KEYS=(list, []),
     ENABLE_AUTOMATIC_ATTACHMENT_FILE_DELETION=(bool, True),
     ENABLE_SWAGGER_UI=(bool, True),
+    # Old audit log settings, TODO: remove later in ATV-252.
     ELASTIC_AUDIT_LOG_INDEX=(str, "logs-atv-audit"),
     ELASTIC_CREATE_DATA_STREAM=(bool, False),
     ELASTIC_HOST=(str, ""),
@@ -163,6 +170,7 @@ def sentry_traces_sampler(sampling_context: SamplingContext) -> float:
     "documents",
     "audit_log",
     "logger_extra",
+    "resilient_logger",
 ]
 
 MIDDLEWARE = [
@@ -267,7 +275,7 @@ def sentry_traces_sampler(sampling_context: SamplingContext) -> float:
 }
 
 # Audit logging
-
+# Old audit log settings TODO: remove later in ATV-252.
 AUDIT_LOG_ORIGIN = "atv"
 ELASTIC_AUDIT_LOG_INDEX = env("ELASTIC_AUDIT_LOG_INDEX")
 ELASTIC_HOST = env("ELASTIC_HOST")
@@ -279,6 +287,32 @@ def sentry_traces_sampler(sampling_context: SamplingContext) -> float:
 ENABLE_SEND_AUDIT_LOG = env("ENABLE_SEND_AUDIT_LOG")
 CLEAR_AUDIT_LOG_ENTRIES = env("CLEAR_AUDIT_LOG_ENTRIES")
 
+# Resilient logger settings
+RESILIENT_LOGGER = {
+    "origin": "atv",
+    "environment": env("AUDIT_LOG_ENVIRONMENT"),
+    "sources": [
+        {
+            "class": "resilient_logger.sources.ResilientLogSource",
+        }
+    ],
+    "targets": [
+        {
+            "class": "resilient_logger.targets.ElasticsearchLogTarget",
+            "es_url": env("AUDIT_LOG_ES_URL"),
+            "es_username": env("AUDIT_LOG_ES_USERNAME"),
+            "es_password": env("AUDIT_LOG_ES_PASSWORD"),
+            "es_index": env("AUDIT_LOG_ES_INDEX"),
+            "required": True,
+        }
+    ],
+    "batch_limit": 5000,
+    "chunk_size": 500,
+    "submit_unsent_entries": True,
+    "clear_sent_entries": True,
+}
+
+
 USE_X_FORWARDED_FOR = env("USE_X_FORWARDED_FOR")
 
 LOGGING = {

audit_log/README.md

@@ -1,14 +1,14 @@
+import logging
 from datetime import datetime, timezone
-from typing import Callable, Optional, Union
+from typing import Optional, Union
 
-from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import AnonymousUser
 from django.db.models import Model
 from django.db.models.base import ModelBase
+from resilient_logger.sources import ResilientLogSource
 
 from audit_log.enums import Operation, Role, Status
-from audit_log.models import AuditLogEntry
 
 User = get_user_model()
 
@@ -18,18 +18,12 @@ def _now() -> datetime:
     return datetime.now(tz=timezone.utc)
 
 
-def _iso8601_date(time: datetime) -> str:
-    """Formats the timestamp in ISO-8601 format, e.g. '2020-06-01T00:00:00.000Z'."""
-    return f"{time.replace(tzinfo=None).isoformat(timespec='milliseconds')}Z"
-
-
 def log(
     actor: Optional[Union[User, AnonymousUser]],
     actor_backend: str,
     operation: Operation,
     target: Union[Model, ModelBase],
     status: Status = Status.SUCCESS,
-    get_time: Callable[[], datetime] = _now,
     ip_address: str = "",
     additional_information: str = "",
     service=None,
@@ -43,7 +37,6 @@ def log(
     an operation(e.g. READ or UPDATE), the target of the operation
     (a Django model instance), status (e.g. SUCCESS), and a timestamp.
     """
-    current_time = get_time()
     user_id_field_name = getattr(actor, "audit_log_id_field", "pk")
     user_id = str(getattr(actor, user_id_field_name, None) or "")
 
@@ -70,24 +63,20 @@ def log(
             "API-Key" if hasattr(actor, "service_api_key") else "Token"
         )
     target_id = _get_target_id(target)
-    message = {
-        "audit_event": {
-            "origin": settings.AUDIT_LOG_ORIGIN,
-            "status": status.value,
-            "date_time_epoch": int(current_time.timestamp() * 1000),
-            "date_time": _iso8601_date(current_time),
-            "actor": actor_data,
-            "operation": operation.value,
-            "additional_information": additional_information,
-            "target": {
-                "type": _get_target_type(target),
-                "id": target_id,
-                "lookup_field": lookup_field if target_id else "",
-                "endpoint": view_name,
-            },
+
+    ResilientLogSource.create_structured(
+        level=logging.NOTSET,
+        message=status.value,
+        actor=actor_data,
+        operation=operation.value,
+        target={
+            "type": _get_target_type(target),
+            "id": target_id,
+            "lookup_field": lookup_field if target_id else "",
+            "endpoint": view_name,
         },
-    }
-    AuditLogEntry.objects.create(message=message)
+        extra={"status": status, "additional_information": additional_information},
+    )
 
 
 def _get_target_id(target: Union[Model, ModelBase]) -> str: