fixup! Improve handling of fail queues (#6781)

Author: achave11-ucsc

scripts/reindex.py

@@ -144,7 +144,7 @@ def main(argv: list[str]):
     reject(args.deindex and (args.delete or args.create),
            '--deindex is incompatible with --create and --delete.')
 
-    assert azul.is_valid_failed_queues_state(), R(
+    assert azul.is_valid_fail_queues_state(), R(
         'Operation canceled due to non-empty failed queues. Running manage_queues.py is '
         'required to empty the queues.')
 
@@ -189,7 +189,7 @@ def main(argv: list[str]):
                             args.prefix, args.catalogs)
             else:
                 azul.wait_for_indexer()
-                assert azul.is_valid_failed_queues_state(), R(
+                assert azul.is_valid_fail_queues_state(), R(
                     'Operation failed due to non empty failed queues. Use manage_queues.py to '
                     'amend this state before retrying.')
 

src/azul/azulclient.py

@@ -600,13 +600,14 @@ def message(source: SourceRef):
         messages = map(message, sources)
         self.queue_mirror_messages(messages)
 
-    def is_valid_failed_queues_state(self) -> bool:
-        queues = set()
-        for failed_queue in config.fail_queue_names:
-            log.info('Checking whether queue %r is empty', failed_queue)
-            queues.add(self.is_queue_empty(failed_queue))
-        assert len(queues) > 0, R('Failed queues not checked', queues)
-        return all(queues)
+    def is_valid_fail_queues_state(self) -> bool:
+        queue_states= set()
+        for fail_queue in config.fail_queue_names:
+            queue_state = self.is_queue_empty(fail_queue)
+            queue_states.add(queue_state)
+            log.info('Queue %r is empty: %r', fail_queue, queue_state)
+        assert len(queue_states) > 0, R('No fail queues found')
+        return all(queue_states)
 
 
 class AzulClientError(RuntimeError):

test/integration_test.py

@@ -430,7 +430,7 @@ class Catalog:
 
         def _wait_for_indexer():
             self.azul_client.wait_for_indexer()
-            self._empty_failed_queues_check()
+            self._empty_fail_queues_check()
 
         flags = config.it_flags
         index, delete = ['no_' + flag not in flags for flag in ['index', 'delete']]
@@ -440,7 +440,7 @@ def _wait_for_indexer():
         else:
             log.warning('Will skip indexing due to overriding IT flag.')
 
-        self._empty_failed_queues_check()
+        self._empty_fail_queues_check()
         catalogs: list[Catalog] = []
         for catalog in config.integration_test_catalogs:
             if index:
@@ -498,7 +498,7 @@ def _wait_for_indexer():
             self._test_mirroring()
 
     def _reset_indexer(self):
-        self._empty_failed_queues_check()
+        self._empty_fail_queues_check()
         # While it's OK to erase the integration test catalog, the queues are
         # shared by all catalogs and we can't afford to trash them in a stable
         # deployment like production.
@@ -509,8 +509,8 @@ def _reset_indexer(self):
                                        delete_indices=True,
                                        create_indices=True)
 
-    def _empty_failed_queues_check(self):
-        assert self.azul_client.is_valid_failed_queues_state(), R(
+    def _empty_fail_queues_check(self):
+        assert self.azul_client.is_valid_fail_queues_state(), R(
             'Failed queues are not empty. Use manage_queues.py to amend this state '
             'before retrying.')