backups: Handle transiently incorrect partition count

When using the backups library to restore topics on KRaft, we are
observing the partition count check failing. Subsequent manual checks of
the partition count shows the correct count. This commit adds a fix to
make the restoration process anticipate that partition count metadata
will be initially incorrect but eventually converge on the correct
value.

Author: aiven-anton

src/karapace/backup/api.py

@@ -7,6 +7,8 @@
 
 from __future__ import annotations
 
+import time
+
 from .backends.reader import BaseBackupReader, BaseItemsBackupReader, ProducerSend, RestoreTopic, RestoreTopicLegacy
 from .backends.v3.constants import V3_MARKER
 from .backends.v3.schema import ChecksumAlgorithm
@@ -272,7 +274,23 @@ def _producer(config: Config, topic: str) -> Iterator[KafkaProducer]:
     :raises Exception: if client creation fails, concrete exception types are unknown, see Kafka implementation.
     """
     with kafka_producer_from_config(config) as producer:
-        __check_partition_count(topic, producer.partitions_for)
+        partition_count_exceptions = []
+        for attempt in range(5):
+            try:
+                __check_partition_count(topic, producer.partitions_for)
+            except PartitionCountError as exception:
+                LOG.info(
+                    "Partition count for topic %r is not yet as expected, will wait and retry: %r",
+                    topic,
+                    exception,
+                )
+                partition_count_exceptions.append(exception)
+                time.sleep(10)
+            else:
+                break
+        else:
+            LOG.warning("Partition count still incorrect after exhausting all retry attempts.")
+            raise exception  # noqa: F821
         yield producer