[change] Replace third-party JSONField with native models.JSONField and optimize indexing #673

Modernized JSON storage using native Django fields and PostgreSQL GIN indexes. Includes SeparateDatabaseAndState for safe schema migration, SpatiaLite compatibility guards, and enhanced CI health-checks.

Closes #673

Author: HMTAhir327

.github/workflows/ci.yml

@@ -89,9 +89,26 @@ jobs:
           sudo npm install -g prettier
 
       - name: Start InfluxDB
+        id: start_influxdb
         if: ${{ !cancelled() && steps.deps.conclusion == 'success' }}
         run: docker compose up -d influxdb
 
+      - name: Wait for InfluxDB
+        if: ${{ !cancelled() && steps.deps.conclusion == 'success' && steps.start_influxdb.conclusion == 'success' }}
+        run: |
+          for i in {1..20}; do
+            if [ "$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8086/ping)" = "204" ]; then
+              echo "InfluxDB is ready!"
+              exit 0
+            fi
+            echo "Waiting for InfluxDB to initialize..."
+            if [ "$i" -lt 20 ]; then
+              sleep 3
+            fi
+          done
+          echo "Error: InfluxDB timed out"
+          exit 1
+
       - name: QA checks
         run: |
           ./run-qa-checks

openwisp_monitoring/monitoring/migrations/0013_auto_20260224_0730.py

@@ -0,0 +1,92 @@
+import django.contrib.postgres.indexes
+from django.db import migrations, models
+
+
+def migrate_to_jsonb(apps, schema_editor):
+    if schema_editor.connection.vendor != "postgresql":
+        return
+    with schema_editor.connection.cursor() as cursor:
+        cursor.execute(
+            "DROP INDEX IF EXISTS monitoring_metric_main_tags_2d8550ae_like; "
+            "DROP INDEX IF EXISTS monitoring_metric_main_tags_2d8550ae;"
+        )
+        cursor.execute(
+            "ALTER TABLE monitoring_metric ALTER COLUMN main_tags TYPE jsonb USING "
+            "CASE WHEN main_tags IS NULL OR btrim(main_tags) = '' THEN '{}'::jsonb "
+            "ELSE main_tags::jsonb END; "
+            "ALTER TABLE monitoring_metric ALTER COLUMN extra_tags TYPE jsonb USING "
+            "CASE WHEN extra_tags IS NULL OR btrim(extra_tags) = '' THEN '{}'::jsonb "
+            "ELSE extra_tags::jsonb END;"
+        )
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS main_tags_gin_idx ON monitoring_metric USING gin (main_tags); "
+            "CREATE INDEX IF NOT EXISTS extra_tags_gin_idx ON monitoring_metric USING gin (extra_tags);"
+        )
+
+
+def rollback_jsonb(apps, schema_editor):
+    if schema_editor.connection.vendor != "postgresql":
+        return
+    with schema_editor.connection.cursor() as cursor:
+        cursor.execute(
+            "DROP INDEX IF EXISTS main_tags_gin_idx; DROP INDEX IF EXISTS extra_tags_gin_idx;"
+        )
+        cursor.execute(
+            "ALTER TABLE monitoring_metric ALTER COLUMN main_tags TYPE text; "
+            "ALTER TABLE monitoring_metric ALTER COLUMN extra_tags TYPE text;"
+        )
+        cursor.execute(
+            "CREATE INDEX IF NOT EXISTS monitoring_metric_main_tags_2d8550ae "
+            "ON monitoring_metric (main_tags); "
+            "CREATE INDEX IF NOT EXISTS monitoring_metric_main_tags_2d8550ae_like "
+            "ON monitoring_metric (main_tags text_pattern_ops);"
+        )
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("monitoring", "0012_migrate_signal_metrics"),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            state_operations=[
+                migrations.AlterField(
+                    model_name="metric",
+                    name="extra_tags",
+                    field=models.JSONField(
+                        blank=True,
+                        default=dict,
+                        verbose_name="extra tags",
+                    ),
+                ),
+                migrations.AlterField(
+                    model_name="metric",
+                    name="main_tags",
+                    field=models.JSONField(
+                        blank=True,
+                        db_index=True,
+                        default=dict,
+                        verbose_name="main tags",
+                    ),
+                ),
+                migrations.AddIndex(
+                    model_name="metric",
+                    index=django.contrib.postgres.indexes.GinIndex(
+                        fields=["main_tags"],
+                        name="main_tags_gin_idx",
+                    ),
+                ),
+                migrations.AddIndex(
+                    model_name="metric",
+                    index=django.contrib.postgres.indexes.GinIndex(
+                        fields=["extra_tags"],
+                        name="extra_tags_gin_idx",
+                    ),
+                ),
+            ],
+            database_operations=[
+                migrations.RunPython(migrate_to_jsonb, reverse_code=rollback_jsonb),
+            ],
+        ),
+    ]

openwisp_monitoring/monitoring/migrations/0013_migrate_to_jsonfield.py

@@ -3,3 +3,6 @@ django-redis~=6.0.0
 mock-ssh-server~=0.9.1
 channels_redis~=4.3.0
 freezegun~=1.5.5
+commitizen~=4.13.0  # required by openwisp-qa-check commit validation
+prompt-toolkit!=3.0.52  # avoid questionary regression (questionary#454), fixed in 2.1.1+
+django-filter>=24.0,<25.0  # 25.x removed deprecated DRF schema helpers used in our stack

requirements-test.txt

@@ -1,5 +1,6 @@
 # Generated by Django 3.2.15 on 2022-09-08 11:11
 
+import django.contrib.postgres.indexes
 from django.db import migrations, models
 
 
@@ -19,4 +20,23 @@ class Migration(migrations.Migration):
                 max_length=16,
             ),
         ),
+        migrations.SeparateDatabaseAndState(
+            state_operations=[
+                migrations.AddIndex(
+                    model_name="metric",
+                    index=django.contrib.postgres.indexes.GinIndex(
+                        fields=["main_tags"],
+                        name="main_tags_gin_idx",
+                    ),
+                ),
+                migrations.AddIndex(
+                    model_name="metric",
+                    index=django.contrib.postgres.indexes.GinIndex(
+                        fields=["extra_tags"],
+                        name="extra_tags_gin_idx",
+                    ),
+                ),
+            ],
+            database_operations=[],
+        ),
     ]