Find potential deadlocks using tryLock in profiler shutdown

Author: Baraa-Hasheesh

src/mutex.cpp

@@ -13,6 +13,10 @@ Mutex::Mutex() {
     pthread_mutex_init(&_mutex, &attr);
 }
 
+bool Mutex::tryLock() {
+    return pthread_mutex_trylock(&_mutex) == 0;
+}
+
 void Mutex::lock() {
     pthread_mutex_lock(&_mutex);
 }

src/mutex.h

@@ -17,6 +17,7 @@ class Mutex {
   public:
     Mutex();
 
+    bool tryLock();
     void lock();
     void unlock();
 };

src/profiler.cpp

@@ -68,6 +68,7 @@ static Instrument instrument;
 
 static ProfilingWindow profiling_window;
 
+static volatile bool _in_jfrsync = false;
 
 struct MethodSample {
     u64 samples;
@@ -1211,7 +1212,9 @@ Error Profiler::start(Arguments& args, bool reset) {
     switchLibraryTrap(true);
 
     if (args._output == OUTPUT_JFR) {
+        _in_jfrsync = args._jfr_sync != NULL;
         error = _jfr.start(args, reset);
+        _in_jfrsync = false;
         if (error) {
             uninstallTraps();
             switchLibraryTrap(false);
@@ -2024,6 +2027,27 @@ Error Profiler::restart(Arguments& args) {
 }
 
 void Profiler::shutdown(Arguments& args) {
+    // Potential deadlock may happen between current thread & profiling thread due to usage of jfrsync
+    // To avoid that we use `tryLock` rather than `lock`
+    if (!_state_lock.tryLock()) {
+        volatile bool sleep = true;
+
+        // peek lock until acquired or _in_jfrsync is set
+        while (!_state_lock.tryLock() && !_in_jfrsync) {
+            OS::uninterruptibleSleep(10000000, &sleep); // 10ms
+        }
+
+        // retry to confirm if real deadlock
+        for (int i = 0; i < 10 && !_state_lock.tryLock() && _in_jfrsync; i++) {
+            OS::uninterruptibleSleep(10000000, &sleep);
+        }
+    }
+
+    // deadlock detected, skip stopping the profiler.
+    if (!_state_lock.tryLock() && _in_jfrsync) {
+        Log::warn("%s", "async-profiler deadlock detected during process shutdown, skipping the shutdown hooks");
+        return;
+    }
     MutexLocker ml(_state_lock);
 
     // The last chance to dump profile before VM terminates