Ingest 2024 Spells, Monsters and Items (open5e#748)

* add markdown file source

* split markdown by object type

* Add spells, plus class stubs to satisfy pk links

* Add monsters draft

* add weapons and armor, expose property type value

* fix weapon property filtering issue

* add mundane items and itemsets

* split items into logical sections for easier parsing

* update importer process to handle circular dependencies and provide better erros

* convert items data into new md and json formats

* fix filter issue & item rarities

* fix tests

* Add animals

* add pytest to default test command

* fix item set references for 2024

* tidy up gitignore file

* add mundane adventuring gear

* remove services from items

* rename to srd-2024

* fix references to old key

* cleanup empty test file

* update services with additional values

* update tests with document structure changes

* typo in feats md

* initial draft of spellcasting options

* Update tests

* really remove artificer I swear this time

Author: eepMoody

.gitignore

@@ -1,24 +1,34 @@
+# Environment and configuration files
 local.env
+*.local
+.python-version
+
+# SSL certificates
 selfsigned.*
 
+# OS-specific files
 **/.DS_Store
+
+# Python
 __pycache__
-staticfiles/
+
+# Database
 db\.sqlite3
+db.sqlite3-journal
 
+# Django static files
 staticfiles/
 
+# Search index
 server/whoosh_index/
 
-*.local
-
+# IDE and editor configurations
 .vscode/
-
-# pycharm config
 .idea/
 
-# pyenv env file
-.python-version
-
+# Test files
 api/tests/approved_files/*.recieved.*
+
+# Generated files
 openapi-schema.yml
+

api/management/commands/quicksetup.py

@@ -7,6 +7,7 @@
 
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
+from django.db import IntegrityError
 
 
 class Command(BaseCommand):
@@ -38,7 +39,6 @@ def handle(self, *args, **options):
         else:
             self.stdout.write('Directory is clean')
 
-        """Main logic."""
         self.stdout.write('Migrating the database...')
         migrate_db()
 
@@ -47,7 +47,16 @@ def handle(self, *args, **options):
 
         if settings.INCLUDE_V1_DATA:
             self.stdout.write('Populating the v1 database...')
-            import_v1()
+            try:
+                import_v1()
+            except IntegrityError as e:
+                self.stdout.write(self.style.ERROR(
+                    f'V1 data import failed: Foreign key constraint error: {e}'
+                ))
+                self.stdout.write(self.style.ERROR(
+                    'QUICKSETUP FAILED - Fix foreign key constraint violations before proceeding.'
+                ))
+                return  # Exit without showing "API setup complete"
 
             if not options['noindex']:
                 if settings.BUILD_V1_INDEX:
@@ -59,14 +68,23 @@ def handle(self, *args, **options):
 
         if settings.INCLUDE_V2_DATA:
             self.stdout.write('Populating the v2 database...')
-            import_v2()
+            try:
+                import_v2()
+            except IntegrityError as e:
+                self.stdout.write(self.style.ERROR(
+                    f'V2 data import failed: Foreign key constraint error: {e}'
+                ))
+                self.stdout.write(self.style.ERROR(
+                    'QUICKSETUP FAILED - Fix foreign key constraint violations before proceeding.'
+                ))
+                return  # Exit without showing "API setup complete"
 
-        if not options['noindex']:
-            if settings.BUILD_V2_INDEX:
-                self.stdout.write('Building the v2 index with both v1 and v2 data.')
-                build_v1v2_searchindex()
-        else:
-            self.stdout.write('Skipping v2 index build because of --noindex.')
+            if not options['noindex']:
+                if settings.BUILD_V2_INDEX:
+                    self.stdout.write('Building the v2 index with both v1 and v2 data.')
+                    build_v1v2_searchindex()
+            else:
+                self.stdout.write('Skipping v2 index build because of --noindex.')
 
         self.stdout.write(self.style.SUCCESS('API setup complete.'))
 

api/management/commands/test.py

@@ -0,0 +1,24 @@
+from django.core.management.base import BaseCommand
+import sys
+import pytest
+
+class Command(BaseCommand):
+    help = 'Runs tests using pytest instead of Django\'s default test runner'
+
+    def add_arguments(self, parser):
+        parser.add_argument('args', nargs='*')
+
+    def handle(self, *args, **options):
+        # Convert Django's test command arguments to pytest arguments
+        pytest_args = list(args)
+        
+        if not pytest_args:
+            # If no specific test paths are provided, run all tests
+            pytest_args = ['api/tests', 'api_v2/tests']
+
+        # Add -v for more verbose output by default
+        if '-v' not in pytest_args:
+            pytest_args.insert(0, '-v')
+
+        # Run pytest with the collected arguments
+        sys.exit(pytest.main(pytest_args)) 

api_v2/management/commands/import.py

@@ -1,11 +1,11 @@
-
-
 import os
 import json
 import glob
 
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
+from django.db import IntegrityError, connection
+from django.core import serializers
 
 class Command(BaseCommand):
     """Implementation for the `manage.py `dumpbyorg` subcommand."""
@@ -17,6 +17,9 @@ def add_arguments(self, parser):
                             "--dir",
                             type=str,
                             help="Directory to write files to.")
+        parser.add_argument("--debug",
+                            action="store_true",
+                            help="Load files one by one to identify problematic data.")
 
     def handle(self, *args, **options) -> None:
         self.stdout.write('Checking if directory exists.')
@@ -28,5 +31,151 @@ def handle(self, *args, **options) -> None:
             exit(0)
 
         fixture_filepaths = glob.glob(options['dir'] + '/**/*.json', recursive=True)
+        
+        self.stdout.write(f'Found {len(fixture_filepaths)} fixture files to load.')
+        
+        if options['debug']:
+            self._load_files_individually(fixture_filepaths)
+        else:
+            try:
+                # Disable foreign key checks during loading
+                self._disable_foreign_key_checks()
+                self.stdout.write('Disabled foreign key constraints for loading.')
+
+                call_command('loaddata', fixture_filepaths)
+                
+                # Re-enable foreign key checks and validate
+                self._enable_foreign_key_checks()
+                self.stdout.write('Re-enabled foreign key constraints.')
+                
+                # Check constraints after all data is loaded
+                self._check_constraints()
+                self.stdout.write('All foreign key constraints validated successfully.')
+                
+            except IntegrityError as e:
+                # Re-enable foreign key checks even if there was an error
+                self._enable_foreign_key_checks()
+                
+                self.stdout.write(self.style.ERROR(
+                    f'Data import completed but foreign key constraint validation failed.'
+                ))
+                self.stdout.write(self.style.ERROR(f'Error details: {e}'))
+                
+                # Extract and format the violation details for a clear worklist
+                if "Foreign key constraint violations found:" in str(e):
+                    violations = str(e).split("Foreign key constraint violations found:\n")[1]
+                    self.stdout.write(self.style.ERROR('\nFOREIGN KEY CONSTRAINT VIOLATIONS - WORKLIST:'))
+                    for violation in violations.split('\n'):
+                        if violation.strip():
+                            self.stdout.write(self.style.ERROR(f'  • {violation}'))
+                
+                self.stdout.write(self.style.ERROR(
+                    '\nData import FAILED due to foreign key constraint violations.'
+                ))
+                self.stdout.write(self.style.WARNING(
+                    'Fix the above violations and re-run the import.'
+                ))
+                raise
+
+    def _disable_foreign_key_checks(self):
+        """Disable foreign key constraint checking."""
+        with connection.cursor() as cursor:
+            if connection.vendor == 'sqlite':
+                cursor.execute('PRAGMA foreign_keys = OFF;')
+            elif connection.vendor == 'mysql':
+                cursor.execute('SET foreign_key_checks = 0;')
+            elif connection.vendor == 'postgresql':
+                cursor.execute('SET session_replication_role = replica;')
+
+    def _enable_foreign_key_checks(self):
+        """Re-enable foreign key constraint checking."""
+        with connection.cursor() as cursor:
+            if connection.vendor == 'sqlite':
+                cursor.execute('PRAGMA foreign_keys = ON;')
+            elif connection.vendor == 'mysql':
+                cursor.execute('SET foreign_key_checks = 1;')
+            elif connection.vendor == 'postgresql':
+                cursor.execute('SET session_replication_role = DEFAULT;')
+
+    def _check_constraints(self):
+        """Check all foreign key constraints after loading."""
+        try:
+            with connection.cursor() as cursor:
+                if connection.vendor == 'sqlite':
+                    cursor.execute('PRAGMA foreign_key_check;')
+                    violations = cursor.fetchall()
+                    if violations:
+                        violation_details = []
+                        for violation in violations:
+                            violation_details.append(f"Table: {violation[0]}, Row: {violation[1]}, Parent: {violation[2]}, FK: {violation[3]}")
+                        raise IntegrityError(f"Foreign key constraint violations found:\n" + "\n".join(violation_details))
+                    pass
+        except Exception as e:
+            raise IntegrityError(f"Foreign key constraint validation failed: {e}")
+
+    def _load_files_individually(self, fixture_filepaths):
+        """Load fixture files one by one to identify which one causes the foreign key error."""
+        loaded_count = 0
+        
+        for filepath in sorted(fixture_filepaths):
+            try:
+                self.stdout.write(f'Loading: {filepath}')
+                call_command('loaddata', filepath)
+                loaded_count += 1
+            except IntegrityError as e:
+                self.stdout.write(self.style.ERROR(
+                    f'FOREIGN KEY CONSTRAINT FAILED in file: {filepath}'
+                ))
+                self.stdout.write(self.style.ERROR(f'Error: {e}'))
+                
+                # Try to identify the specific problematic object
+                self._analyze_problematic_file(filepath)
+                
+                self.stdout.write(f'Successfully loaded {loaded_count} files before failure.')
+                raise
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(
+                    f'Other error in file {filepath}: {e}'
+                ))
+                raise
 
-        call_command('loaddata', fixture_filepaths)
+    def _analyze_problematic_file(self, filepath):
+        """Analyze the problematic file to identify specific objects causing issues."""
+        try:
+            with open(filepath, 'r') as f:
+                data = json.load(f)
+            
+            self.stdout.write(f'Analyzing {len(data)} objects in {filepath}:')
+            
+            for i, obj in enumerate(data):
+                try:
+                    # Try to deserialize just this one object
+                    for deserialized_obj in serializers.deserialize('json', [obj]):
+                        # Try to save it
+                        deserialized_obj.save()
+                except IntegrityError as e:
+                    self.stdout.write(self.style.ERROR(
+                        f'  Object {i+1}: {obj.get("model", "unknown")} '
+                        f'pk="{obj.get("pk", "unknown")}" FAILED'
+                    ))
+                    self.stdout.write(self.style.ERROR(f'    Error: {e}'))
+                    
+                    # Show the object's foreign key fields
+                    fields = obj.get('fields', {})
+                    fk_fields = {}
+                    for field_name, field_value in fields.items():
+                        if field_value and (isinstance(field_value, str) or isinstance(field_value, list)):
+                            # Likely a foreign key reference
+                            fk_fields[field_name] = field_value
+                    
+                    if fk_fields:
+                        self.stdout.write(f'    Foreign key fields: {fk_fields}')
+                    
+                    return  # Stop at first problematic object
+                except Exception as e:
+                    self.stdout.write(self.style.WARNING(
+                        f'  Object {i+1}: Other error - {e}'
+                    ))
+                    
+        except Exception as e:
+            self.stdout.write(self.style.ERROR(f'Could not analyze file: {e}'))

api_v2/migrations/0050_spellcastingoption_desc.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.2.1 on 2025-06-08 03:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api_v2', '0049_add_document_display_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='spellcastingoption',
+            name='desc',
+            field=models.TextField(blank=True, help_text='Description of complex casting option effects that cannot be captured in other fields.', null=True),
+        ),
+    ]

api_v2/models/spell.py

@@ -189,3 +189,8 @@ class SpellCastingOption(models.Model):
 
     shape_size = distance_field()
     # Null values mean this value is unchanged from the default casting option.
+    
+    desc = models.TextField(
+        null=True,
+        blank=True,
+        help_text='Description of complex casting option effects that cannot be captured in other fields.')

api_v2/tests/responses/TestObjects.test_spell_cantrip_example.approved.json

@@ -4,6 +4,7 @@
         {
             "concentration": null,
             "damage_roll": null,
+            "desc": null,
             "duration": null,
             "range": null,
             "shape_size": null,