Handle tasks run by another crank turner then replaced by new tasks

tuktuk-crank-turner/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tuktuk-crank-turner"
-version = "0.2.25"
+version = "0.2.26"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true

tuktuk-crank-turner/src/task_processor.rs

@@ -5,9 +5,10 @@ use solana_sdk::{instruction::InstructionError, signer::Signer, transaction::Tra
 use solana_transaction_utils::{error::Error as TransactionQueueError, queue::TransactionTask};
 use tokio_graceful_shutdown::SubsystemHandle;
 use tracing::{debug, info};
-use tuktuk_program::TaskQueueV0;
-use tuktuk_sdk::compiled_transaction::{
-    next_available_task_ids_excluding_in_progress, run_ix_with_free_tasks,
+use tuktuk_program::{TaskQueueV0, TaskV0};
+use tuktuk_sdk::{
+    client::GetAnchorAccount,
+    compiled_transaction::{next_available_task_ids_excluding_in_progress, run_ix_with_free_tasks},
 };
 
 use crate::{
@@ -271,6 +272,23 @@ impl TimedTask {
                 )) if code == 3012 && ctx.rpc_client.get_account(&self.task_key).await.is_err() => {
                     info!(?self.task_key, "task not found, removing from queue");
                 }
+                // Handle task not ready
+                // There's a race condition where the task was already completed and replaced with a new one,
+                // yet we're still trying to process the old task.
+                TransactionQueueError::TransactionError(TransactionError::InstructionError(
+                    _,
+                    InstructionError::Custom(code),
+                )) if code == 6005
+                    && ctx
+                        .rpc_client
+                        .anchor_account::<TaskV0>(&self.task_key)
+                        .await
+                        .ok()
+                        .flatten()
+                        .is_some_and(|acc| acc.queued_at != self.task.queued_at) =>
+                {
+                    info!(?self.task_key, "task was already replaced, skipping");
+                }
                 TransactionQueueError::RawTransactionError(_)
                 | TransactionQueueError::SimulatedTransactionError(_)
                 | TransactionQueueError::TransactionError(_)