Support negative indexing json arrays on mysql/maria

Author: coleifer

CHANGELOG.md

@@ -9,6 +9,12 @@ https://github.com/coleifer/peewee/releases
 
 * Lateral joins honor a user-supplied `on=` predicate instead of silently
   replacing it with `true`.
+* `JSONField` negative path indexes render as `$[last]` / `$[last-n]` on
+  MySQL/MariaDB. Previously the sqlite-only `$[#-n]` form was emitted, which
+  MariaDB evaluates to NULL (overwriting the column when used with `set()`)
+  and MySQL rejects as an invalid path.
+* Reflection/pwiz map MySQL JSON columns to the core `JSONField` instead of
+  emitting `from playhouse.mysql_ext import *` for a re-exported field.
 
 [View commits](https://github.com/coleifer/peewee/compare/4.2.6...master)
 

peewee.py

@@ -3142,11 +3142,14 @@ def _contains_value(self, field, value):
         return Value(field._dumps(value), converter=False)
 
     @staticmethod
-    def _path(keys, suffix=''):
+    def _path_index(idx):
+        return '[#%d]' % idx if idx < 0 else '[%d]' % idx
+
+    def _path(self, keys, suffix=''):
         parts = ['$']
         for k in keys:
             if isinstance(k, int):
-                parts.append('[#%d]' % k if k < 0 else '[%d]' % k)
+                parts.append(self._path_index(k))
             else:
                 k = str(k).replace('\\', '\\\\').replace('"', '\\"')
                 parts.append('."%s"' % k)
@@ -3415,6 +3418,12 @@ def wrapper(value):
                                  compact=False)
         return wrapper
 
+    @staticmethod
+    def _path_index(idx):
+        if idx < 0:
+            return '[last]' if idx == -1 else '[last%d]' % (idx + 1)
+        return '[%d]' % idx
+
     def extract(self, field, keys):
         if not keys:
             return field
@@ -3476,7 +3485,7 @@ def update(self, field, value):
         return fn.JSON_MERGE_PATCH(field, self._json_value(field, value))
 
     def contains(self, field, keys, value):
-        # JSON_CONTAINS returns 0/1; wrap in `= 1` so it composes cleanly in
+        # JSON_CONTAINS returns 0/1, wrap in `= 1` so it composes cleanly in
         # boolean contexts (NOT, AND, etc.).
         path_args = (self._path(keys),) if keys else ()
         call = fn.JSON_CONTAINS(field, self._contains_value(field, value),

playhouse/reflection.py

@@ -12,10 +12,6 @@
 from peewee import _query_val_transform
 from peewee import CommaNodeList
 from peewee import make_snake_case
-try:
-    from playhouse import mysql_ext
-except ImportError:
-    mysql_ext = None
 try:
     from playhouse import postgres_ext
 except ImportError:
@@ -429,7 +425,6 @@ class MySQLMetadata(Metadata):
         'varchar': CharField,
         'year': IntegerField,
     }
-    extension_import = 'from playhouse.mysql_ext import *'
 
     def get_column_types(self, table, schema=None):
         column_types = {}
@@ -442,9 +437,8 @@ def get_column_types(self, table, schema=None):
         for name, data_type, column_type in cursor.fetchall():
             if column_type == 'tinyint(1)':
                 column_types[name] = BooleanField
-            elif data_type.lower() == 'json' and mysql_ext is not None:
-                column_types[name] = mysql_ext.JSONField
-                self.requires_extension = True
+            elif data_type.lower() == 'json':
+                column_types[name] = JSONField
             else:
                 column_types[name] = self.column_map.get(data_type.lower(),
                                                          UnknownField)

tests/json_field.py

@@ -178,10 +178,12 @@ def test_bracket(self):
         self.assertEqual(JM.select(JM.data['tags'][2]).scalar(), 'fluffy')
         self.assertEqual(JM.select(JM.data['matrix'][1][0]).scalar(), 3)
 
-    @skip_if(IS_MYSQL)
     def test_negative_index(self):
         self.assertEqual(JM.select(JM.data['tags'][-1]).scalar(), 'fluffy')
+        self.assertEqual(JM.select(JM.data['tags'][-2]).scalar(), 'white')
         self.assertEqual(JM.select(JM.data['matrix'][-1][-1]).scalar(), 6)
+        JM.update(data=JM.data['tags'][-1].set('spotted')).execute()
+        self.assertEqual(JM.select(JM.data['tags'][-1]).scalar(), 'spotted')
 
     def test_special_char_key(self):
         self.assertEqual(self._val('a.b'), 'dotted key')

tests/reflection.py

@@ -9,6 +9,7 @@
 from .base import IS_CRDB
 from .base import IS_CYSQLITE
 from .base import IS_MYSQL
+from .base import IS_ORACLE_MYSQL
 from .base import IS_POSTGRESQL
 from .base import IS_SQLITE
 from .base import IS_SQLITE_OLD
@@ -186,6 +187,20 @@ def test_sqlite_fk_re(self):
                 'col_types_id', 'coltypes', 'f11',
             ))
 
+    @skip_if(not IS_ORACLE_MYSQL, 'requires mysql (mariadb reports longtext)')
+    def test_mysql_json_maps_to_core_field(self):
+        self.database.execute_sql('drop table if exists rj')
+        self.database.execute_sql(
+            'create table rj (id integer not null primary key '
+            'auto_increment, data json)')
+        try:
+            models = self.introspector.generate_models(table_names=['rj'])
+            field = models['rj']._meta.fields['data']
+            self.assertTrue(isinstance(field, JSONField))
+            self.assertFalse(self.introspector.metadata.requires_extension)
+        finally:
+            self.database.execute_sql('drop table rj')
+
     def test_make_column_name(self):
         # Tests for is_foreign_key=False.
         tests = (