Don't leak extra fds into recorded processes

We are pretty good about setting CLOEXEC on most file descriptors.
However, we were missing them on the pagemap fd and the tty fd.
With lots of nested detaching, this can start filling up fd space,
which manifested as a CI failure on julia CI: JuliaLang/julia#58979

Fix this by setting CLOEXEC as appropriate and add a test to make sure
this doesn't regress.

Author: Keno

CMakeLists.txt

@@ -716,7 +716,7 @@ if (intel_pt_decoding)
 endif()
 
 add_executable(rr ${RR_SOURCES})
-target_compile_definitions(rr PRIVATE 
+target_compile_definitions(rr PRIVATE
   "FULL_LIBDIR=\"${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}\"")
 set_target_properties(rr PROPERTIES ENABLE_EXPORTS true)
 post_build_executable(rr)
@@ -1745,6 +1745,7 @@ set(TESTS_WITHOUT_PROGRAM
   exec_stop
   execp
   explicit_checkpoint_clone
+  fd_leak
   file_name_newline
   final_sigkill
   first_instruction
@@ -2020,7 +2021,7 @@ if(BUILD_TESTS)
       bash source_dir/src/test/${test}.run ${testname} -n bin_dir ${TEST_MONITOR_DEFAULT_TIMEOUT})
     configure_test(${test}-no-syscallbuf)
   endforeach(test)
-  
+
   # Run 32-bit tests on 64-bit builds.
   # We copy the test files into '32' subdirectories in the output
   # directory, so we can set different compile options on them.

src/AutoRemoteSyscalls.cc

@@ -724,6 +724,7 @@ template <typename Arch> ScopedFd AutoRemoteSyscalls::retrieve_fd_arch(int fd) {
     // pidfd_getfd requires a threadgroup leader, so find one if we can.
     Task* tg_leader_for_fds = thread_group_leader_for_fds(t);
     if (tg_leader_for_fds) {
+      // N.B.: pidfd_open fds are always cloexec
       pid_fd = ScopedFd(::syscall(NativeArch::pidfd_open, tg_leader_for_fds->tid, 0));
       ASSERT(t, pid_fd.is_open() || errno == ENOSYS)
         << "Error in pidfd_open errno=" << errno_name(errno);

src/Task.cc

@@ -2993,7 +2993,7 @@ void Task::open_mem_fd_if_needed() {
 
 ScopedFd& Task::pagemap_fd() {
   if (!as->pagemap_fd().is_open()) {
-    ScopedFd fd(proc_pagemap_path().c_str(), O_RDONLY);
+    ScopedFd fd(proc_pagemap_path().c_str(), O_RDONLY | O_CLOEXEC);
     if (fd.is_open()) {
       as->set_pagemap_fd(std::move(fd));
     } else {

src/record_syscall.cc

@@ -6538,7 +6538,7 @@ static bool is_rr_terminal(const string& pathname) {
 static int dev_tty_fd() {
   static int fd = -1;
   if (fd < 0) {
-    fd = open("/dev/tty", O_WRONLY);
+    fd = open("/dev/tty", O_WRONLY | O_CLOEXEC);
   }
   return fd;
 }

src/test/fd_leak.run

@@ -0,0 +1,17 @@
+source `dirname $0`/util.sh
+num_baseline=$(ls -la /proc/self/fd | wc -l)
+just_record $(which rr) "record --nested=detach ls -la /proc/self/fd"
+replay
+num_replay=$(cat replay.out | wc -l)
+# We allow for two extra fds in the recorded process,
+# one for the RR fd 999 and one for the perf_events
+# fd opened by the syscall buf (if present).
+expected_extra_fds=2
+if [[ "-n" == "$LIB_ARG" ]]; then
+expected_extra_fds=$((expected_extra_fds - 1))
+fi
+if [[ $num_replay -gt $((num_baseline + expected_extra_fds)) || $num_replay -lt $num_baseline ]]; then
+    failed
+else
+    passed
+fi