Merge pull request #1899 from MTG/sound-reindex-logging

Add progress indicator and time remaining to index process

Author: ffont

freesound/logger.py

@@ -58,7 +58,7 @@
         },
         'search': {
             'handlers': ['stdout'],
-            'level': 'INFO',
+            'level': 'ERROR',
             'propagate': False,
         },
         'file_upload': {

search/management/commands/post_dirty_sounds_to_search_engine.py

@@ -18,7 +18,9 @@
 #     See AUTHORS file.
 #
 
+import datetime
 import logging
+import time
 
 from sounds.models import Sound
 from utils.management_commands import LoggingBaseCommand
@@ -28,12 +30,23 @@
 console_logger = logging.getLogger("console")
 
 
+def time_stats(done, total, starttime):
+    nowtime = time.monotonic()
+    position = done*1.0 / total
+    duration = round(nowtime - starttime)
+    durdelta = datetime.timedelta(seconds=duration)
+    remaining = round((duration / position) - duration)
+    remdelta = datetime.timedelta(seconds=remaining)
+
+    return str(durdelta), str(remdelta)
+
+
 def send_sounds_to_search_engine(sounds_to_index_ids, slice_size=4000, delete_if_existing=False):
-    num_sounds = len(sounds_to_index_ids)
-    console_logger.info("Starting to post dirty sounds to solr. %i sounds to be added/updated to the search engine"
-                        % num_sounds)
+    total_sounds = len(sounds_to_index_ids)
+    console_logger.info(f"Starting to post dirty sounds to solr. {total_sounds} sounds to be added/updated to the search engine")
     n_sounds_indexed_correctly = 0
-    for i in range(0, num_sounds, slice_size):
+    starttime = time.monotonic()
+    for i in range(0, total_sounds, slice_size):
         sound_ids_slice = sounds_to_index_ids[i:i + slice_size]
         if delete_if_existing:
             delete_sounds_from_search_engine(sound_ids_slice)
@@ -42,6 +55,8 @@ def send_sounds_to_search_engine(sounds_to_index_ids, slice_size=4000, delete_if
         if n_sounds_indexed > 0:
             Sound.objects.filter(pk__in=sound_ids_slice).update(is_index_dirty=False)
         n_sounds_indexed_correctly += n_sounds_indexed
+        elapsed, remaining = time_stats(n_sounds_indexed_correctly, total_sounds, starttime)
+        console_logger.info(f"Added {n_sounds_indexed_correctly}/{total_sounds} sounds. Elapsed: {elapsed}, Remaining: {remaining}")
 
     return n_sounds_indexed_correctly
 

utils/search/search_sounds.py

@@ -75,18 +75,11 @@ def perform_search_engine_query(query_params):
     return results, paginator
 
 
-def add_sounds_to_search_engine(sound_objects, fields_to_include=[], update=False):
+def add_sounds_to_search_engine(sound_objects: list["sounds.models.Sound"]):
     """Add the Sounds from the queryset to the search engine
 
     Args:
-        sound_objects (list[sounds.models.Sound]): list (or queryset) of Sound objects to index
-        fields_to_include (list[str]): use this list to indicate the specific field names of the sounds 
-            that need to be included in the documents that will be indexed. If no fields are specified 
-            (fields_to_update=[]), then all available fields will be included.
-        update (bool): if True, the sounds' data will be updated in the index, otherwise it will be 
-            replaced by the new generated documents. This is specially useful in combination with
-            fields_to_include so that different fields of the indexed can be updated separately. 
-
+        sound_objects: list (or queryset) of Sound objects to index
     Returns:
         int: number of sounds added to the index
     """
@@ -95,13 +88,13 @@ def add_sounds_to_search_engine(sound_objects, fields_to_include=[], update=Fals
     else:
         num_sounds = len(sound_objects)
     try:
-        console_logger.info("Adding %d sounds to the search engine" % num_sounds)
-        search_logger.info("Adding %d sounds to the search engine" % num_sounds)
-        get_search_engine().add_sounds_to_index(sound_objects, fields_to_include=fields_to_include, update=update)
+        console_logger.debug(f"Adding {num_sounds} sounds to the search engine")
+        search_logger.debug(f"Adding {num_sounds} sounds to the search engine")
+        get_search_engine().add_sounds_to_index(sound_objects, fields_to_include=[], update=False)
         return num_sounds
     except SearchEngineException as e:
-        console_logger.info(f"Failed to add sounds to search engine index: {str(e)}")
-        search_logger.info(f"Failed to add sounds to search engine index: {str(e)}")
+        console_logger.error(f"Failed to add sounds to search engine index: {str(e)}")
+        search_logger.error(f"Failed to add sounds to search engine index: {str(e)}")
         return 0