Skip to content

feat: Switch to JSONField for SDK metrics labels #5833

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
+96 −11
Open

feat: Switch to JSONField for SDK metrics labels #5833

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8c22c5b
feat: Switch to `JSONField` for SDK metrics labels
khvn26 Jul 25, 2025
e0789c3
fixup the original migration
khvn26 Jul 25, 2025
091006e
add `USING`, conditional `ALTER`s
khvn26 Aug 6, 2025
6c848c6
docs
khvn26 Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions api/app_analytics/migrations/0006_add_labels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# Generated by Django 4.2.21 on 2025-06-16 16:55

import django.contrib.postgres.fields.hstore
from django.contrib.postgres.operations import HStoreExtension
# import django.contrib.postgres.fields.hstore
# from django.contrib.postgres.operations import HStoreExtension

from django.db import migrations
from django.db import models


class Migration(migrations.Migration):
Expand All @@ -13,25 +14,37 @@ class Migration(migrations.Migration):
]

operations = [
HStoreExtension(),
# # The extension usage is annulated by the subsequent migration.
# # To avoid the operational overhead of enabling it in some environments,
# # we do not create it here anymore.
# # However, leave it commented out for reference/history.
# HStoreExtension(),
migrations.AddField(
model_name="apiusagebucket",
name="labels",
field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
# This field is now a JSONField instead of HStoreField.
# field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
field=models.JSONField(default=dict),
),
migrations.AddField(
model_name="apiusageraw",
name="labels",
field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
# This field is now a JSONField instead of HStoreField.
# field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
field=models.JSONField(default=dict),
),
migrations.AddField(
model_name="featureevaluationbucket",
name="labels",
field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
# This field is now a JSONField instead of HStoreField.
# field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
field=models.JSONField(default=dict),
),
migrations.AddField(
model_name="featureevaluationraw",
name="labels",
field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
# This field is now a JSONField instead of HStoreField.
# field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
field=models.JSONField(default=dict),
),
]
73 changes: 73 additions & 0 deletions api/app_analytics/migrations/0007_labels_jsonb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# Generated by Django 4.2.22 on 2025-07-25 14:09

from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("app_analytics", "0006_add_labels"),
]

operations = [
migrations.SeparateDatabaseAndState(
state_operations=[
migrations.AlterField(
model_name="apiusagebucket",
name="labels",
field=models.JSONField(default=dict),
),
migrations.AlterField(
model_name="apiusageraw",
name="labels",
field=models.JSONField(default=dict),
),
migrations.AlterField(
model_name="featureevaluationbucket",
name="labels",
field=models.JSONField(default=dict),
),
migrations.AlterField(
model_name="featureevaluationraw",
name="labels",
field=models.JSONField(default=dict),
),
],
database_operations=[
migrations.RunSQL(
# Only alter the columns that are currently hstore.
# See 0006_add_labels to understand why we are doing this.
sql="""
DO $$
BEGIN
FOR relname IN
SELECT c.relname
FROM pg_class c
JOIN pg_attribute a ON a.attrelid = c.oid
JOIN pg_type t ON a.atttypid = t.oid
WHERE c.relname IN (
'app_analytics_apiusagebucket',
'app_analytics_apiusageraw',
'app_analytics_featureevaluationbucket',
'app_analytics_featureevaluationraw'
)
AND a.attname = 'labels'
AND t.typname = 'hstore'
LOOP
EXECUTE format(
'ALTER TABLE %I
ALTER COLUMN labels TYPE jsonb USING hstore_to_json(labels),
ALTER COLUMN labels SET DEFAULT ''{}''::jsonb',
relname
);
END LOOP;
END
$$;
""",
# We don't want hstore in the database at all,
# so don't do anything for reverse SQL.
reverse_sql="",
),
Comment on lines +37 to +70
Copy link
Contributor

@emyller emyller Aug 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The SQL code looks fine, though I think we should benefit from the ORM abstraction until we need to fall back to raw SQL.

Here's an untested brain dump.

Given:

def upgrade_field(model_name: str, field_name: str) -> Callable[[Apps, SchemaEditor], None]:
    def upgrade(apps: Apps, schema_editor: SchemaEditor) -> None:
        model = apps.get_model("app_analytics", model_name)
        field = model._meta.get_field(field_name)
        if isinstance(field, models.JSONField):
            continue
        schema_editor.execute(f"""
            ALTER TABLE {model._meta.db_table}
            ALTER COLUMN {field.column} TYPE jsonb USING hstore_to_json({field.column}),
            ALTER COLUMN {field.column} SET DEFAULT '{{}}'::jsonb
        """)
    return upgrade

Suggested replacement for the SQL script:

Suggested change
migrations.RunSQL(
# Only alter the columns that are currently hstore.
# See 0006_add_labels to understand why we are doing this.
sql="""
DO $$
BEGIN
FOR relname IN
SELECT c.relname
FROM pg_class c
JOIN pg_attribute a ON a.attrelid = c.oid
JOIN pg_type t ON a.atttypid = t.oid
WHERE c.relname IN (
'app_analytics_apiusagebucket',
'app_analytics_apiusageraw',
'app_analytics_featureevaluationbucket',
'app_analytics_featureevaluationraw'
)
AND a.attname = 'labels'
AND t.typname = 'hstore'
LOOP
EXECUTE format(
'ALTER TABLE %I
ALTER COLUMN labels TYPE jsonb USING hstore_to_json(labels),
ALTER COLUMN labels SET DEFAULT ''{}''::jsonb',
relname
);
END LOOP;
END
$$;
""",
# We don't want hstore in the database at all,
# so don't do anything for reverse SQL.
reverse_sql="",
),
migrations.RunPython(
code=upgrade_field("apiusagebucket", "labels"),
reverse_code=migrations.RunPython.noop,
),
migrations.RunPython(
code=upgrade_field("apiusageraw", "labels"),
reverse_code=migrations.RunPython.noop,
),
migrations.RunPython(
code=upgrade_field("featureevaluationbucket", "labels"),
reverse_code=migrations.RunPython.noop,
),
migrations.RunPython(
code=upgrade_field("featureevaluationraw", "labels"),
reverse_code=migrations.RunPython.noop,
),

],
),
]
7 changes: 3 additions & 4 deletions api/app_analytics/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from datetime import timedelta

from django.contrib.postgres.fields import HStoreField
from django.core.exceptions import ValidationError
from django.db import models
from django_lifecycle import ( # type: ignore[import-untyped]
Expand Down Expand Up @@ -59,7 +58,7 @@ class APIUsageRaw(models.Model):
host = models.CharField(max_length=255)
resource = models.IntegerField(choices=Resource.choices)
count = models.PositiveIntegerField(default=1)
labels = HStoreField(default=dict)
labels = models.JSONField(default=dict)

class Meta:
index_together = (("environment_id", "created_at"),)
Expand All @@ -70,7 +69,7 @@ class AbstractBucket(LifecycleModelMixin, models.Model): # type: ignore[misc]
created_at = models.DateTimeField()
total_count = models.PositiveIntegerField()
environment_id = models.PositiveIntegerField()
labels = HStoreField(default=dict)
labels = models.JSONField(default=dict)

class Meta:
abstract = True
Expand Down Expand Up @@ -107,7 +106,7 @@ class FeatureEvaluationRaw(models.Model):
environment_id = models.PositiveIntegerField()
evaluation_count = models.IntegerField(default=0)
created_at = models.DateTimeField(auto_now_add=True)
labels = HStoreField(default=dict)
labels = models.JSONField(default=dict)

# Both stored for tracking multivariate split testing.
identity_identifier = models.CharField(max_length=2000, null=True, default=None)
Expand Down
Loading