merged from staging

Author: calumbell

.gitignore

@@ -32,3 +32,4 @@ api/tests/approved_files/*.recieved.*
 # Generated files
 openapi-schema.yml
 
+server/vector_index.pkl

api_v2/admin.py

@@ -100,6 +100,13 @@ class LanguageAdmin(admin.ModelAdmin):
 
 admin.site.register(Condition)
 
+class ConditionConceptAdmin(admin.ModelAdmin):
+    list_display = ['key', 'name']
+    filter_horizontal = ['conditions']
+    search_fields = ['key', 'name', 'desc']
+    
+admin.site.register(ConditionConcept, ConditionConceptAdmin)
+
 admin.site.register(ClassFeatureItem)
 admin.site.register(ClassFeature, admin_class=ClassFeatureAdmin)
 admin.site.register(CharacterClass)

api_v2/management/commands/export.py

@@ -124,10 +124,11 @@ def handle(self, *args, **options) -> None:
 
                 for model in app_models:
                     SKIPPED_MODEL_NAMES = ['Document', 'GameSystem', 'License', 'Publisher','SearchResult']
+                    CONCEPT_MODEL_NAMES = ['ConditionConcept']  # These are synthetic/concept models not tied to documents
                     CHILD_MODEL_NAMES = ['SpeciesTrait', 'FeatBenefit', 'BackgroundBenefit', 'ClassFeatureItem', 'SpellCastingOption','CreatureAction', 'CreatureTrait']
                     CHILD_CHILD_MODEL_NAMES = ['CreatureActionAttack']
 
-                    if model._meta.app_label == 'api_v2' and model.__name__ not in SKIPPED_MODEL_NAMES:
+                    if model._meta.app_label == 'api_v2' and model.__name__ not in SKIPPED_MODEL_NAMES and model.__name__ not in CONCEPT_MODEL_NAMES:
                         modelq=None
                         if model.__name__ in CHILD_CHILD_MODEL_NAMES:
                             modelq = model.objects.filter(parent__parent__document=doc).order_by('pk')
@@ -147,6 +148,23 @@ def handle(self, *args, **options) -> None:
                 self.stdout.write(self.style.SUCCESS(
                     'Wrote {} to {}'.format(doc.key, doc_path)))
 
+        # Export concept models (synthetic objects that aggregate across systems)
+        concept_models = ['ConditionConcept']  # Add other concept models here as they're created
+        for concept_model_name in concept_models:
+            try:
+                model = apps.get_model('api_v2', concept_model_name)
+                concept_queryset = model.objects.all().order_by('pk')
+                concept_path = get_filepath_by_model(
+                    concept_model_name,
+                    'api_v2',
+                    base_path=options['dir'],
+                    format=options['format'])
+                write_queryset_data(concept_path, concept_queryset, format=options['format'])
+                self.stdout.write(self.style.SUCCESS(f'Exported {concept_model_name} concept objects'))
+            except LookupError:
+                # Model doesn't exist yet, skip it
+                self.stdout.write(self.style.WARNING(f'Concept model {concept_model_name} not found, skipping'))
+
         self.stdout.write(self.style.SUCCESS('Data for v2 data complete.'))
 
 
@@ -158,7 +176,7 @@ def get_filepath_by_model(model_name, app_label, pub_key=None, doc_key=None, bas
 
     if app_label == "api_v2":
         root_folder_name = 'v2'
-        root_models = ['License', 'GameSystem']
+        root_models = ['License', 'GameSystem', 'ConditionConcept']  # Concept models are exported at root level
         pub_models = ['Publisher']
 
         if model_name in root_models:
@@ -192,7 +210,7 @@ def write_queryset_data(filepath, queryset, format='json'):
 
         with open(output_filepath, 'w', encoding='utf-8') as f:
             if format=='json':
-                serializers.serialize("json", queryset, indent=2, stream=f)
+                serializers.serialize("json", queryset, indent=2, stream=f, sort_keys=True)
             if format=='csv':
                 # Create headers:
                 fieldnames = []

api_v2/management/commands/import.py

@@ -17,6 +17,9 @@ def add_arguments(self, parser):
                             "--dir",
                             type=str,
                             help="Directory to write files to.")
+        parser.add_argument("--skip-concepts",
+                            action="store_true",
+                            help="Skip automatic concept population after import.")
         parser.add_argument("--debug",
                             action="store_true",
                             help="Load files one by one to identify problematic data.")
@@ -139,6 +142,15 @@ def _load_files_individually(self, fixture_filepaths):
                 ))
                 raise
 
+        call_command('loaddata', fixture_filepaths)
+        
+        # After loading data, populate concept objects to aggregate equivalent content across systems
+        if not options.get('skip_concepts', False):
+            self.stdout.write(self.style.SUCCESS('Data import complete. Now populating concept aggregations...'))
+            call_command('populate_concepts')
+        else:
+            self.stdout.write('Skipping concept population (--skip-concepts flag provided)')
+
     def _analyze_problematic_file(self, filepath):
         """Analyze the problematic file to identify specific objects causing issues."""
         try:

api_v2/management/commands/populate_concepts.py

@@ -0,0 +1,75 @@
+"""
+Management command to populate concept objects by grouping equivalent content across game systems.
+
+This command analyzes existing content objects and groups them by name to create
+concept objects that represent the same conceptual item across different game systems.
+
+Currently supports:
+- ConditionConcept (for Condition objects)
+
+This can be extended to support other content types like SpellConcept, ItemConcept, etc.
+"""
+
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.utils.text import slugify
+from collections import defaultdict
+
+from api_v2.models import Condition, ConditionConcept
+
+
+class Command(BaseCommand):
+    help = 'Populate concept objects by grouping equivalent content across game systems'
+
+    def handle(self, *args, **options):
+        self.populate_condition_concepts()
+        
+        # Future content types can be added here:
+        # self.populate_damage_type_concepts()
+        # self.populate_environment_concepts()
+
+    def populate_condition_concepts(self):
+        """Populate ConditionConcept objects by grouping equivalent conditions."""
+        self.stdout.write(self.style.HTTP_INFO('Processing conditions...'))
+        
+        # Always clear existing concepts and reconstruct
+        self.stdout.write('Clearing existing ConditionConcept objects...')
+        ConditionConcept.objects.all().delete()
+
+        # Group conditions by exact name (no normalization needed)
+        condition_groups = defaultdict(list)
+        
+        for condition in Condition.objects.all():
+            condition_groups[condition.name].append(condition)
+
+        self.stdout.write(f'Found {len(condition_groups)} condition concepts to create:')
+
+        created_count = 0
+        
+        with transaction.atomic():
+            for concept_name, conditions in condition_groups.items():
+                concept_key = slugify(concept_name)
+                
+                # Create description based on number of systems
+                systems = list(set([c.document.gamesystem.name for c in conditions]))
+                if len(systems) == 1:
+                    concept_desc = f"The {concept_name.lower()} condition from {systems[0]}."
+                else:
+                    concept_desc = f"The {concept_name.lower()} condition as it appears across different game systems: {', '.join(systems)}."
+                
+                self.stdout.write(f'  - {concept_name} ({concept_key}): {len(conditions)} conditions across {len(systems)} systems')
+                
+                # Create the ConditionConcept
+                concept = ConditionConcept.objects.create(
+                    key=concept_key,
+                    name=concept_name,
+                    desc=concept_desc
+                )
+                
+                # Add all conditions to this concept
+                concept.conditions.set(conditions)
+                created_count += 1
+
+        self.stdout.write(self.style.SUCCESS(f'Successfully created {created_count} ConditionConcept objects'))
+
+ 

api_v2/migrations/0042_add_condition_concept.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.2.1 on 2025-06-07 17:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api_v2', '0041_merge_20250509_0736'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ConditionConcept',
+            fields=[
+                ('name', models.CharField(help_text='Name of the item.', max_length=100)),
+                ('desc', models.TextField(help_text='Description of the game content item. Markdown.')),
+                ('key', models.CharField(help_text="Unique key for the condition concept (e.g., 'invisible').", max_length=100, primary_key=True, serialize=False)),
+                ('conditions', models.ManyToManyField(help_text='All condition implementations that are equivalent to this concept.', related_name='concepts', to='api_v2.condition')),
+            ],
+            options={
+                'verbose_name': 'condition concept',
+                'verbose_name_plural': 'condition concepts',
+                'ordering': ['name'],
+            },
+        ),
+    ]

api_v2/migrations/0050_merge_20250608_0235.py

@@ -0,0 +1,14 @@
+# Generated by Django 5.2.1 on 2025-06-08 02:35
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api_v2', '0042_add_condition_concept'),
+        ('api_v2', '0049_add_document_display_name'),
+    ]
+
+    operations = [
+    ]

api_v2/migrations/0051_merge_20250615_2056.py

@@ -0,0 +1,14 @@
+# Generated by Django 5.2.1 on 2025-06-15 20:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api_v2', '0050_merge_20250608_0235'),
+        ('api_v2', '0050_spellcastingoption_desc'),
+    ]
+
+    operations = [
+    ]

api_v2/models/__init__.py

@@ -42,6 +42,7 @@
 from .alignment import Alignment
 
 from .condition import Condition
+from .condition import ConditionConcept
 
 from .spell import Spell
 from .spell import SpellCastingOption

api_v2/models/condition.py

@@ -1,5 +1,6 @@
 """The model for a condition."""
 from django.db import models
+from django.db.models import Q
 
 from .abstracts import HasName, HasDescription
 from .document import FromDocument
@@ -14,3 +15,52 @@ class Meta:
         """To assist with the UI layer."""
 
         verbose_name_plural = "conditions"
+
+
+class ConditionConcept(HasName, HasDescription):
+    """
+    This model represents a synthetic condition concept that aggregates
+    equivalent conditions across different game systems.
+    
+    For example, "Invisible" may exist in multiple game systems (SRD 2014, A5e, etc.)
+    but conceptually they represent the same condition. This model provides a 
+    unified view of that concept with links to all its implementations.
+    """
+    
+    key = models.CharField(
+        primary_key=True,
+        max_length=100,
+        help_text="Unique key for the condition concept (e.g., 'invisible')."
+    )
+    
+    conditions = models.ManyToManyField(
+        Condition,
+        related_name='concepts',
+        help_text="All condition implementations that are equivalent to this concept."
+    )
+    
+    @property
+    def gamesystems(self):
+        """Returns a list of all game systems that have this condition concept."""
+        return list(set([condition.document.gamesystem for condition in self.conditions.all()]))
+    
+    @property
+    def documents(self):
+        """Returns a list of all documents that have this condition concept."""
+        return list(set([condition.document for condition in self.conditions.all()]))
+    
+    def get_condition_for_gamesystem(self, gamesystem_key):
+        """
+        Returns the condition for a specific game system.
+        If multiple conditions exist for the same game system, returns the first one.
+        """
+        return self.conditions.filter(document__gamesystem__key=gamesystem_key).first()
+    
+    def get_condition_for_document(self, document_key):
+        """Returns the condition for a specific document."""
+        return self.conditions.filter(document__key=document_key).first()
+    
+    class Meta:
+        verbose_name = "condition concept"
+        verbose_name_plural = "condition concepts"
+        ordering = ['name']