Fix two deadlocks in SubprocessScheduler

Running the HHBBC compiler against a trivial test program in recent OSS
HHVM hangs forever.

LLDB shows workers deadlocking in SubprocessScheduler.
One thread is waiting on `m_subprocessSchedulingCV` called when a
subprocess is released:
```
* thread facebook#5, name = 'HPHPcWorker2'
  * frame #0: 0x00007ffff436d9a2 libc.so.6`__syscall_cancel_arch_end
    frame #1: 0x00007ffff4361c3c libc.so.6`__internal_syscall_cancel + 92
    frame facebook#2: 0x00007ffff43622ac libc.so.6`__futex_abstimed_wait_common + 124
    frame facebook#3: 0x00007ffff436497e libc.so.6`pthread_cond_wait@@GLIBC_2.3.2 + 334
    frame facebook#4: 0x00007ffff46e60f3 libc++.so.1`std::__1::condition_variable::wait(std::__1::unique_lock<std::__1::mutex>&) + 19
    frame facebook#5: 0x000000000069dccc hhvm`HPHP::extern_worker::SubprocessScheduler::acquire(this=0x00007fffeea62980, jobName="hhbbc-build-subclass") at subprocess-scheduler.cpp:56:30
    frame facebook#6: 0x00000000006593b8 hhvm`HPHP::extern_worker::(anonymous namespace)::SubprocessImpl::exec(this=<unavailable>, requestId=<unavailable>, command=<unavailable>, config=<unavailable>, inputs=<unavailable>, output=<unavailable>, finiOutput=<unavailable>, (null)=<unavailable>) (.resume) at extern-worker.cpp:1540:44
    frame facebook#7: 0x00000000042d1891 hhvm`std::__1::coroutine_handle<void>::resume[abi:se210108](this=<unavailable>) const at coroutine_handle.h:69:5 [inlined]
    frame facebook#8: 0x00000000042d1881 hhvm`folly::resumeCoroutineWithNewAsyncStackRoot(h=coro frame = 0x7fffcc625000, frame=<unavailable>) at AsyncStack.cpp:202:5
    frame facebook#9: 0x000000000061e3b0 hhvm`folly::coro::detail::co_reschedule_on_current_executor_::StackAwareAwaiter::await_suspend_impl(std::__1::coroutine_handle<void>, folly::AsyncStackFrame&)::'lambda'()::operator()(this=0x00007fffd5f75270) at CurrentExecutor.h:92:15
    frame facebook#10: 0x00000000041f54a2 hhvm`folly::detail::function::FunctionTraits<void ()>::operator()(this=0x00007fffd5f75270) at Function.h:370:12 [inlined]
    frame facebook#11: 0x00000000041f5499 hhvm`void folly::catch_exception<folly::Function<void ()>&, void (&)(char const*) noexcept, char const*&, void>(t=0x00007fffd5f75270, c=<unavailable>, a=<unavailable>) at Exception.h:359:12 [inlined]
    frame facebook#12: 0x00000000041f5499 hhvm`void folly::Executor::invokeCatchingExns<folly::Function<void ()>>(p=<unavailable>, f=Function<void ()> @ 0x00007fffd5f75270) at Executor.h:233:5 [inlined]
    frame facebook#13: 0x00000000041f5499 hhvm`folly::ThreadPoolExecutor::runTask(this=0x00007fffee6cc800, thread=std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>::element_type @ 0x00007fffd9217140, task=0x00007fffd5f75410) at ThreadPoolExecutor.cpp:142:7
    frame facebook#14: 0x00000000041d3c9b hhvm`folly::CPUThreadPoolExecutor::threadRun(this=0x00007fffee6cc800, thread=<unavailable>) at CPUThreadPoolExecutor.cpp:337:5
    frame facebook#15: 0x00000000041f7533 hhvm`std::__1::__invoke_result_impl<void, void (folly::ThreadPoolExecutor::*&)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__1::__invoke[abi:se210108]<void (folly::ThreadPoolExecutor::*&)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(__args=<unavailable>, __args=<unavailable>, __args=<unavailable>) at invoke.h:87:27 [inlined]
    frame facebook#16: 0x00000000041f74fd hhvm`std::__1::__bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>, __is_valid_bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>>::value>::type std::__1::__apply_functor[abi:se210108]<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, 0ul, 1ul, std::__1::tuple<>>(__f=<unavailable>, __bound_args=<unavailable>, (null)=<unavailable>, __args=<unavailable>) at bind.h:195:10 [inlined]
    frame facebook#17: 0x00000000041f74fd hhvm`std::__1::__bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>, __is_valid_bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>>::value>::type std::__1::__bind<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::operator()[abi:se210108]<>(this=<unavailable>) at bind.h:222:12 [inlined]
    frame facebook#18: 0x00000000041f74fd hhvm`void folly::detail::function::call_<std::__1::__bind<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>, true, false, void>(p=<unavailable>) at Function.h:341:5
    frame facebook#19: 0x0000000000642d11 hhvm`folly::detail::function::FunctionTraits<void ()>::operator()(this=<unavailable>) at Function.h:370:12 [inlined]
    frame facebook#20: 0x0000000000642d0d hhvm`HPHP::(anonymous namespace)::ThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()::operator()(this=0x00007fffd9254050) at coro.cpp:53:9 [inlined]
    frame facebook#21: 0x0000000000642cf2 hhvm`void folly::detail::function::call_<HPHP::(anonymous namespace)::ThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'(), false, false, void>(p=<unavailable>) at Function.h:341:5
    frame facebook#22: 0x0000000000644770 hhvm`folly::detail::function::FunctionTraits<void ()>::operator()(this=<unavailable>) at Function.h:370:12 [inlined]
    frame facebook#23: 0x000000000064476d hhvm`folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()::operator()(this=<unavailable>) at NamedThreadFactory.h:40:11 [inlined]
    frame facebook#25: 0x000000000064476d hhvm`void std::__1::__thread_execute[abi:se210108]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()>(__t=size=2, (null)=<unavailable>) at thread.h:159:3 [inlined]
    frame facebook#26: 0x0000000000644749 hhvm`void* std::__1::__thread_proxy[abi:se210108]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()>>(__vp=0x00007fffd9259150) at thread.h:168:3
    frame facebook#27: 0x00007ffff4365464 libc.so.6`start_thread + 740
    frame facebook#28: 0x00007ffff43e85ec libc.so.6`__clone3 + 44
```

However this worker does that while also holding `m_subprocessMapLock`,
so the other worker can't insert a subprocess:
```
* thread facebook#3, name = 'HPHPcWorker0'
  * frame #0: 0x00007ffff43624e0 libc.so.6`__GI___lll_lock_wait + 48
    frame #1: 0x00007ffff4368a31 libc.so.6`__pthread_mutex_lock@GLIBC_2.2.5 + 257
    frame facebook#2: 0x00007ffff46e724a libc++.so.1`std::__1::mutex::lock() + 10
    frame facebook#3: 0x000000000069e074 hhvm`std::__1::lock_guard<std::__1::mutex>::lock_guard[abi:se210108](this=<unavailable>, __m=0x00007fffeea629e8) at lock_guard.h:32:10 [inlined]
    frame facebook#4: 0x000000000069e06c hhvm`HPHP::extern_worker::SubprocessScheduler::registerSubprocess(this=0x00007fffeea62980, pid=35418, jobName=<unavailable>) at subprocess-scheduler.cpp:75:31
    frame facebook#5: 0x0000000000657a40 hhvm`HPHP::extern_worker::(anonymous namespace)::SubprocessImpl::doSubprocess(this=<unavailable>, requestId=<unavailable>, command=<unavailable>, inputBlob=<unavailable>, outputPath=<unavailable>, usageEstimate=<unavailable>) (.resume) at extern-worker.cpp:1657:28
    frame facebook#6: 0x00000000042d1891 hhvm`std::__1::coroutine_handle<void>::resume[abi:se210108](this=<unavailable>) const at coroutine_handle.h:69:5 [inlined]
    frame facebook#7: 0x00000000042d1881 hhvm`folly::resumeCoroutineWithNewAsyncStackRoot(h=coro frame = 0x7ffff33c4000, frame=<unavailable>) at AsyncStack.cpp:202:5
    frame facebook#8: 0x000000000061e3b0 hhvm`folly::coro::detail::co_reschedule_on_current_executor_::StackAwareAwaiter::await_suspend_impl(std::__1::coroutine_handle<void>, folly::AsyncStackFrame&)::'lambda'()::operator()(this=0x00007fffdf6f4270) at CurrentExecutor.h:92:15
    frame facebook#9: 0x00000000041f54a2 hhvm`folly::detail::function::FunctionTraits<void ()>::operator()(this=0x00007fffdf6f4270) at Function.h:370:12 [inlined]
    frame facebook#10: 0x00000000041f5499 hhvm`void folly::catch_exception<folly::Function<void ()>&, void (&)(char const*) noexcept, char const*&, void>(t=0x00007fffdf6f4270, c=<unavailable>, a=<unavailable>) at Exception.h:359:12 [inlined]
    frame facebook#11: 0x00000000041f5499 hhvm`void folly::Executor::invokeCatchingExns<folly::Function<void ()>>(p=<unavailable>, f=Function<void ()> @ 0x00007fffdf6f4270) at Executor.h:233:5 [inlined]
    frame facebook#12: 0x00000000041f5499 hhvm`folly::ThreadPoolExecutor::runTask(this=0x00007fffee6cc800, thread=std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>::element_type @ 0x00007fffdff21240, task=0x00007fffdf6f4410) at ThreadPoolExecutor.cpp:142:7
    frame facebook#13: 0x00000000041d3c9b hhvm`folly::CPUThreadPoolExecutor::threadRun(this=0x00007fffee6cc800, thread=<unavailable>) at CPUThreadPoolExecutor.cpp:337:5
    frame facebook#14: 0x00000000041f7533 hhvm`std::__1::__invoke_result_impl<void, void (folly::ThreadPoolExecutor::*&)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::type std::__1::__invoke[abi:se210108]<void (folly::ThreadPoolExecutor::*&)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*&, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>(__args=<unavailable>, __args=<unavailable>, __args=<unavailable>) at invoke.h:87:27 [inlined]
    frame facebook#15: 0x00000000041f74fd hhvm`std::__1::__bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>, __is_valid_bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>>::value>::type std::__1::__apply_functor[abi:se210108]<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, 0ul, 1ul, std::__1::tuple<>>(__f=<unavailable>, __bound_args=<unavailable>, (null)=<unavailable>, __args=<unavailable>) at bind.h:195:10 [inlined]
    frame facebook#16: 0x00000000041f74fd hhvm`std::__1::__bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>, __is_valid_bind_return<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), std::__1::tuple<folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>>, std::__1::tuple<>>::value>::type std::__1::__bind<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>::operator()[abi:se210108]<>(this=<unavailable>) at bind.h:222:12 [inlined]
    frame facebook#17: 0x00000000041f74fd hhvm`void folly::detail::function::call_<std::__1::__bind<void (folly::ThreadPoolExecutor::*)(std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>), folly::ThreadPoolExecutor*, std::__1::shared_ptr<folly::ThreadPoolExecutor::Thread>&>, true, false, void>(p=<unavailable>) at Function.h:341:5
    frame facebook#18: 0x0000000000642d11 hhvm`folly::detail::function::FunctionTraits<void ()>::operator()(this=<unavailable>) at Function.h:370:12 [inlined]
    frame facebook#19: 0x0000000000642d0d hhvm`HPHP::(anonymous namespace)::ThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()::operator()(this=0x00007fffeeacb240) at coro.cpp:53:9 [inlined]
    frame facebook#20: 0x0000000000642cf2 hhvm`void folly::detail::function::call_<HPHP::(anonymous namespace)::ThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'(), false, false, void>(p=<unavailable>) at Function.h:341:5
    frame facebook#21: 0x0000000000644770 hhvm`folly::detail::function::FunctionTraits<void ()>::operator()(this=<unavailable>) at Function.h:370:12 [inlined]
    frame facebook#22: 0x000000000064476d hhvm`folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()::operator()(this=<unavailable>) at NamedThreadFactory.h:40:11 [inlined]
    frame facebook#24: 0x000000000064476d hhvm`void std::__1::__thread_execute[abi:se210108]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()>(__t=size=2, (null)=<unavailable>) at thread.h:159:3 [inlined]
    frame facebook#25: 0x0000000000644749 hhvm`void* std::__1::__thread_proxy[abi:se210108]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct>>, folly::NamedThreadFactory::newThread(folly::Function<void ()>&&)::'lambda'()>>(__vp=0x00007fffeea23d40) at thread.h:168:3
    frame facebook#26: 0x00007ffff4365464 libc.so.6`start_thread + 740
    frame facebook#27: 0x00007ffff43e85ec libc.so.6`__clone3 + 44
```

So release `m_subprocessMapLock` in `acquire()` once we're done reading
the map so that other threads get a chance to write it.

SubprocessScheduler also sets the memory limit that may be used by
subprocesses to 60% of the system RAM. Before scheduling
a subprocess, it tries to check if the estimated memory usage
of the subprocess will exceed this limit, and block until
another subprocess has freed memory. It starts with an initial
hardcoded 20GiB memory usage estimate that immediately deadlocks
on systems with < ~32GiB RAM as we start blocking on `m_subprocessSchedulingCV`
waiting for subprocesses to free memory before we even scheduled any.
So avoid blocking if no subprocesses have been started yet,
as we'd never be unblocked.

Author: mszabo-wikia

hphp/util/subprocess-scheduler.cpp

@@ -43,14 +43,16 @@ SubprocessScheduler::SubprocessScheduler(uint64_t maxMemoryUsage)
 
 uint64_t SubprocessScheduler::acquire(const std::string& jobName) {
   do {
-    std::lock_guard<std::mutex> _{m_subprocessMapLock};
-    uint64_t usageEstimate = getRSSEstimate(jobName);
-    if (usageEstimate == 0 ||
-        m_estimatedMemoryUsage + usageEstimate <= m_maxMemoryUsage) {
-      FTRACE(3, "Scheduling `{}` job ({} running). est. job RSS: {}, est. total memory usage: {}\n",
-              jobName, m_runningSubprocesses.size(), usageEstimate, m_estimatedMemoryUsage);
-      m_estimatedMemoryUsage += usageEstimate;
-      return usageEstimate;
+    {
+      std::lock_guard<std::mutex> _{m_subprocessMapLock};
+      uint64_t usageEstimate = getRSSEstimate(jobName);
+      if (usageEstimate == 0 ||
+          m_estimatedMemoryUsage + usageEstimate <= m_maxMemoryUsage || m_runningSubprocesses.empty()) {
+        FTRACE(3, "Scheduling `{}` job ({} running). est. job RSS: {}, est. total memory usage: {}\n",
+                jobName, m_runningSubprocesses.size(), usageEstimate, m_estimatedMemoryUsage);
+        m_estimatedMemoryUsage += usageEstimate;
+        return usageEstimate;
+      }
     }
     std::unique_lock<std::mutex> lock(m_subprocessSchedulingLock);
     m_subprocessSchedulingCV.wait(lock);