Merge pull request #441 from open5e/v2_search_improved

V2 search improved

Author: augustjohnson

api_v2/management/commands/index_v1.py renamed to api_v2/management/commands/buildindex.py

@@ -18,44 +18,72 @@ def unload_all_content(self):
         v2.SearchResult.objects.all().delete()
         print("UNLOADED_OBJECT_COUNT:{}".format(object_count))
 
-
-    def load_content(self,model,schema):
-        print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format(
-                    schema,
-                    model.objects.all().count(),
-                    model.__name__,
-                    model._meta.db_table))
-
+    def load_v1_content(self, model):
+        results = []
         standard_v1_models = ['MagicItem','Spell','Monster','CharClass','Archetype',
-                        'Race','Subrace','Plane','Section','Feat','Condition','Background','Weapon','Armor']
-
-        search_results = []
+                'Race','Subrace','Plane','Section','Feat','Condition','Background','Weapon','Armor']
 
-        if model.__name__ in standard_v1_models and schema=='v1':
+        if model.__name__ in standard_v1_models:
             for o in model.objects.all():
-                search_results.append(v2.SearchResult(
+                results.append(v2.SearchResult(
                     document_pk=o.document.slug,
-                    document_name=o.document.title,
                     object_pk=o.slug,
                     object_name=o.name,
-                    object_route=o.route,
+                    object_model=o.__class__.__name__,
                     schema_version="v1",
                     text=o.name+"\n"+o.desc
 
                 ))
+        return results
 
-        v2.SearchResult.objects.bulk_create(search_results)
+    def load_v2_content(self, model):
+        results = []
+        standard_v2_models = ['Item','Spell','Creature','CharacterClass','Race','Feat','Condition','Background']
 
+        if model.__name__ in standard_v2_models:
+            for o in model.objects.all():
+                results.append(v2.SearchResult(
+                    document_pk=o.document.key,
+                    object_pk=o.pk,
+                    object_name=o.name,
+                    object_model=o.__class__.__name__,
+                    schema_version='v2',
+                    text=o.as_text()
+                ))
+        return results
+
+    def load_content(self,model,schema):
+        print("SCHEMA:{} OBJECT_COUNT:{} MODEL:{} TABLE_NAME:{}".format(
+                    schema,
+                    model.objects.all().count(),
+                    model.__name__,
+                    model._meta.db_table))
+
+        if schema == 'v1':
+            v2.SearchResult.objects.bulk_create(
+                self.load_v1_content(model)
+            )
+
+        if schema == 'v2':
+            v2.SearchResult.objects.bulk_create(
+                self.load_v2_content(model)
+            )
 
     def load_index(self):
         with connection.cursor() as cursor:
 
             cursor.execute("DROP TABLE IF EXISTS search_index;")
-            
-            cursor.execute("CREATE VIRTUAL TABLE search_index USING FTS5(document_pk,document_name,object_pk,object_name,object_route,text,schema_version);")
 
-            cursor.execute("INSERT INTO search_index (document_pk,document_name,object_pk,object_name,object_route,text,schema_version) SELECT document_pk,document_name,object_pk,object_name,object_route,text,schema_version FROM api_v2_searchresult")
-    
+            cursor.execute(
+                "CREATE VIRTUAL TABLE search_index " +
+                "USING FTS5(document_pk,object_pk,object_name,object_model,text,schema_version);")
+
+            cursor.execute(
+                "INSERT INTO search_index " +
+                "(document_pk,object_pk,object_name,object_model,text,schema_version) " +
+                "SELECT document_pk,object_pk,object_name,object_model,text,schema_version " +
+                "FROM api_v2_searchresult")
+
 
     def check_fts_enabled(self):
         #import sqlite3
@@ -91,6 +119,15 @@ def handle(self, *args, **options) -> None:
         self.load_content(v1.Weapon,"v1")
         self.load_content(v1.Armor,"v1")
 
+        self.load_content(v2.Item,"v2")
+        self.load_content(v2.Spell,"v2")
+        self.load_content(v2.Creature,"v2")
+        self.load_content(v2.CharacterClass,"v2")
+        self.load_content(v2.Race,"v2")
+        self.load_content(v2.Feat,"v2")
+        self.load_content(v2.Condition,"v2")
+        self.load_content(v2.Background,"v2")
+
 
         # Take the content table's current data and load it into the index.
         self.load_index()

api_v2/migrations/0073_auto_20240403_0010.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.20 on 2024-04-03 00:10
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api_v2', '0072_merge_20240329_1829'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='searchresult',
+            old_name='object_route',
+            new_name='object_model',
+        ),
+        migrations.RemoveField(
+            model_name='searchresult',
+            name='document_name',
+        ),
+    ]

api_v2/models/background.py

@@ -27,3 +27,4 @@ class Meta:
         """To assist with the UI layer."""
 
         verbose_name_plural = "backgrounds"
+

api_v2/models/characterclass.py

@@ -97,4 +97,12 @@ def __str__(self):
         if self.is_subclass:
             return "{} [{}]".format(self.subclass_of.name, self.name)
         else:
-            return self.name
+            return self.name
+
+    def as_text(self):
+        text = self.name + '\n'
+        
+        for feature in self.feature_set.all():
+            text+='\n' + feature.as_text()
+
+        return text

api_v2/models/creature.py

@@ -57,6 +57,21 @@ class Creature(Object, Abilities, FromDocument):
         max_length=100,
         help_text='The creature\'s allowed alignments.'
     )
+    
+    def as_text(self):
+        text = self.name + '\n'
+        
+        for action in self.creatureaction_set.all():
+            text+='\n' + action.as_text()
+
+        return text
+        
+    def search_result_extra_fields(self):
+        return {
+            "armor_class":self.armor_class,
+            "hit_points":self.hit_points,
+            "ability_scores":self.get_ability_scores(),
+              }
 
     @property
     def creatureset(self):

api_v2/models/document.py

@@ -112,8 +112,16 @@ class FromDocument(models.Model):
         max_length=100,
         help_text="Unique key for the Item.")
 
+    def as_text(self):
+        return "{}\n\n{}".format(self.name, self.desc)
+
     def get_absolute_url(self):
         return reverse(self.__name__, kwargs={"pk": self.pk})
 
+    def search_result_extra_fields(self):
+        return {
+            "school":self.school.key,
+        }
+
     class Meta:
         abstract = True

api_v2/models/item.py

@@ -70,6 +70,13 @@ class Item(Object, HasDescription, FromDocument):
     def is_magic_item(self):
         return self.rarity is not None
 
+    def search_result_extra_fields(self):
+        fields = {"type":self.category.key}
+        if self.is_magic_item:
+            fields["rarity"]=self.rarity.key
+        return fields
+
+
 
 class ItemSet(HasName, HasDescription, FromDocument):
     """A set of items to be referenced."""

api_v2/models/search.py

@@ -5,32 +5,11 @@ class SearchResult(models.Model):
     """ The Search Result object model"""
 
     document_pk = models.CharField(max_length=255)
-    document_name = models.CharField(max_length=100)
     object_pk = models.CharField(max_length=255)
     object_name = models.CharField(max_length=100)
-    object_route = models.CharField(max_length=255)
+    object_model = models.CharField(max_length=255)
     schema_version = models.CharField(max_length=100)
 
     rank = models.DecimalField(max_digits=10, decimal_places=4, null=True, default=None)
     text = models.TextField(null=True, default=None)
     highlighted = models.TextField(null=True, default=None)
-
-    @property
-    def document_slug(self):
-        return self.document_pk
-
-    @property
-    def document_title(self):
-        return self.document_name
-
-    @property
-    def route(self):
-        return self.object_route
-
-    @property
-    def slug(self):
-        return self.object_pk
-
-    @property
-    def name(self):
-        return self.object_name

api_v2/serializers/search.py

@@ -4,42 +4,85 @@
 
 from api_v2 import models
 from api import models as v1
+from django.urls import reverse
 
 class SearchResultSerializer(serializers.ModelSerializer):
-    rank = serializers.ReadOnlyField()
-    text = serializers.ReadOnlyField()
-    highlighted = serializers.ReadOnlyField()
     object = serializers.SerializerMethodField(method_name='get_object')
+    document = serializers.SerializerMethodField(method_name='get_document')
+    route = serializers.SerializerMethodField(method_name='get_route')
 
 
     class Meta:
         model = models.SearchResult
         fields = [
-            'document_pk',
-            'document_name',
+            'document',
             'object_pk',
             'object_name',
             'object',
-            'object_route',
+            'object_model',
             'schema_version',
+            'route',
             'rank',
             'text',
             'highlighted']
 
     def get_object(self, obj):
+        result_detail = None
+
         if obj.schema_version == 'v1':
-            if obj.object_route == 'magicitems/':
+            if obj.object_model == 'MagicItem':
                 result_detail = v1.MagicItem.objects.get(slug=obj.object_pk)
-                return result_detail.search_result_extra_fields()
-                
-            if obj.object_route == 'monsters/':
+            if obj.object_model == 'Monster':
                 result_detail = v1.Monster.objects.get(slug=obj.object_pk)
-                return result_detail.search_result_extra_fields()
-                
-            if obj.object_route == 'spells/':
+            if obj.object_model == 'Spell':
                 result_detail = v1.Spell.objects.get(slug=obj.object_pk)
-                return result_detail.search_result_extra_fields()
-
-            if obj.object_route == 'sections/':
+            if obj.object_model == 'Section':
                 result_detail = v1.Section.objects.get(slug=obj.object_pk)
-                return result_detail.search_result_extra_fields()
+
+        if obj.schema_version == 'v2':
+            if obj.object_model == 'Item':
+                result_detail = models.Item.objects.get(pk=obj.object_pk)
+            if obj.object_model == 'Creature':
+                result_detail = models.Creature.objects.get(pk=obj.object_pk)
+            if obj.object_model == 'Spell':
+                result_detail = models.Spell.objects.get(pk=obj.object_pk)
+
+        if result_detail is not None:
+            return result_detail.search_result_extra_fields()
+        else:
+            return None
+
+    def get_document(self, obj):
+        if obj.schema_version == 'v1':
+            doc = v1.Document.objects.get(slug=obj.document_pk)
+            return {
+                'key': doc.slug,
+                'name': doc.title
+                }
+
+        if obj.schema_version == 'v2':
+            doc = models.Document.objects.get(key=obj.document_pk)
+            return {
+                'key': doc.key,
+                'name': doc.name
+                }
+
+    def get_route(self, obj):
+        # May want to split this out into v1 and v2?
+        route_lookup = {
+            "Item":"items",
+            "Creature":"creatures",
+            "Spell":"spells",
+            "CharacterClass":"class",
+            "Monster":"monsters",
+            "MagicItem":"magicitems",
+            "Section":"sections",
+            "Background":"backgrounds",
+            "Subrace":"subraces",
+            "Feat":"feats",
+            "Race":"races",
+            "Plane":"planes",
+        }
+
+        route = "{}/{}/".format(obj.schema_version,route_lookup[obj.object_model])
+        return route

api_v2/views/search.py

@@ -33,20 +33,21 @@ def get_queryset(self):
         else:
             document_pk = self.request.query_params.get("document_pk")
 
-        if self.request.query_params.get("object_route") is None:
-            object_route = '%'
+        if self.request.query_params.get("object_model") is None:
+            object_model = '%'
         else:
-            object_route = self.request.query_params.get("object_route")
+            object_model = self.request.query_params.get("object_model")
 
-        queryset = models.SearchResult.objects.raw(
+        weighted_queryset = models.SearchResult.objects.raw(
             "SELECT 1 as id,rank, " +
-            "snippet(search_index,5,'<span class=\"highlighted\">','</span>','...',20) as highlighted, " + 
-            "* FROM search_index " + 
+            "snippet(search_index,4,'<span class=\"highlighted\">','</span>','...',20) as highlighted, " + 
+            "document_pk,object_pk,object_name,object_model,text,schema_version FROM search_index " + 
             "WHERE " + 
             "schema_version LIKE %s " +
             "AND document_pk LIKE %s " + 
-            "AND object_route LIKE %s " + 
-            "AND text MATCH %s " + 
-            "ORDER BY rank",[schema_version, document_pk, object_route, query])
+            "AND object_model LIKE %s " + 
+            "AND search_index MATCH %s" + 
+            "AND rank MATCH 'bm25(1.0, 1.0, 10.0)'"+ # This line results in a 10x weight to Name
+            "ORDER BY rank",[schema_version, document_pk, object_model, query])
 
-        return queryset
+        return weighted_queryset