Make sure systematic testing doesn't switch to suspended thread (#4647)

* Make sure systematic testing doesn't switch to suspended thread

Prior to this commit, the systematic testing next thread logic
would consider switching to the pinned actor thread even if it
was suspended. This was very wasteful since the suspended pinned
actor thread would not do any meaningful work and almost immediately
re-suspend itself.

This commit changes things so that the next thread logic properly
takes into account whether the pinned actor thread is suspended
or not when selecting the next thread to activate.

* add release notes

Author: dipinhora

.release-notes/4647.md

@@ -0,0 +1,5 @@
+## Make sure systematic testing doesn't switch to suspended thread
+
+Previously, the systematic testing next thread logic would consider switching to the pinned actor thread even if it was suspended. This was very wasteful since the suspended pinned actor thread would not do any meaningful work and almost immediately re-suspend itself.
+
+We have changed things so that the next thread logic properly takes into account whether the pinned actor thread is suspended or not when selecting the next thread to activate.

src/libponyrt/sched/scheduler.c

@@ -1903,6 +1903,14 @@ PONY_API pony_ctx_t* pony_ctx()
   return &this_scheduler->ctx;
 }
 
+/**
+ * Gets whether the pinned actor scheduler is suspended or not
+ */
+bool ponyint_get_pinned_actor_scheduler_suspended()
+{
+   return atomic_load_explicit(&pinned_actor_scheduler_suspended, memory_order_relaxed);
+}
+
 void ponyint_register_asio_thread()
 {
   pony_register_thread();

src/libponyrt/sched/scheduler.h

@@ -135,6 +135,8 @@ void ponyint_sched_start_global_unmute(uint32_t from, pony_actor_t* actor);
 
 bool ponyint_sched_unmute_senders(pony_ctx_t* ctx, pony_actor_t* actor);
 
+bool ponyint_get_pinned_actor_scheduler_suspended();
+
 PONY_API uint32_t pony_active_schedulers();
 
 PONY_API int32_t pony_scheduler_index();

src/libponyrt/sched/systematic_testing.c

@@ -158,11 +158,21 @@ void ponyint_systematic_testing_start(scheduler_t* schedulers, pony_thread_id_t
 static uint32_t get_next_index()
 {
   uint32_t active_scheduler_count = pony_active_schedulers();
-  uint32_t active_count = active_scheduler_count + 2; // account for asio and pinned actor thread
-  uint32_t next_index = 0;
+  bool pinned_actor_scheduler_suspended = ponyint_get_pinned_actor_scheduler_suspended();
+  uint32_t active_count = active_scheduler_count + 1; // account for asio thread
+  // account for pinned actor thread if it is not suspended
+  if(!pinned_actor_scheduler_suspended)
+    active_count = active_count + 1;
+
+  uint32_t next_index = -1;
   do
   {
     next_index = rand() % active_count;
+
+    // skip over pinned actor thread index if it is suspended
+    if(pinned_actor_scheduler_suspended)
+      next_index = next_index + 1;
+
     pony_assert(next_index <= total_threads);
   }
   while (threads_to_track[next_index].stopped);