Make background compile the default

Summary: This makes background compilation the default mode that we run under. We disable it in IG and in tests where it throws off the expectations. We keep it enabled in the `jit-auto` variation and add a `dev` build run pass that runs in this mode (where we were previous missing any significant coverage).

Reviewed By: alexmalyshev

Differential Revision: D114134434

fbshipit-source-id: 1d50c8a8d8572290f19c0234216006e5bb681a57

Author: DinoV

cinderx/Jit/config.h

@@ -205,7 +205,7 @@ struct Config {
   // the GIL, instead of compiling inline on the calling thread.  The calling
   // thread keeps running through the interpreter until the background
   // compilation finishes and swaps in the JIT-compiled entry point.
-  bool background_compile{false};
+  bool background_compile{true};
   // When a function is being compiled, this is the maximum number of dependent
   // functions called by it that can be compiled along with it.
   size_t preload_dependent_limit{99};

cinderx/PythonLib/test_cinderx/test_compiler/test_static/test_compile.py

@@ -5369,7 +5369,10 @@ def f4():
                 "main.py",
             ]
             proc = subprocess.run(
-                cmd, capture_output=True, cwd=str(d), env=subprocess_env()
+                cmd,
+                capture_output=True,
+                cwd=str(d),
+                env={"CINDERX_JIT_BACKGROUND_COMPILE": "0", **subprocess_env()},
             )
             self.assertEqual(proc.returncode, 0, proc.stderr)
 

cinderx/PythonLib/test_cinderx/test_free_threading/test_free_threading.py

@@ -12,6 +12,13 @@
 
 
 class FibonacciTest(unittest.TestCase):
+    def setUp(self):
+        self.bg_compile = cinderx.jit.get_background_compile()
+        cinderx.jit.background_compile(False)
+
+    def tearDown(self):
+        cinderx.jit.background_compile(self.bg_compile)
+
     @unittest.skipUnless(hasattr(os, "fork"), "fork not available on Windows")
     @run_in_subprocess
     def test_concurrent_calls_trigger_jit_compilation(self) -> None:

cinderx/PythonLib/test_cinderx/test_jit_disable.py

@@ -265,7 +265,7 @@ def inc(x):
             subprocess.run(
                 [sys.executable, str(test_file)],
                 check=True,
-                env=subprocess_env(),
+                env={"CINDERX_JIT_BACKGROUND_COMPILE": "0", **subprocess_env()},
             )
 
     def test_auto_predefined(self) -> None:
@@ -298,7 +298,7 @@ def predefined(x):
             subprocess.run(
                 [sys.executable, str(test_file)],
                 check=True,
-                env=subprocess_env(),
+                env={"CINDERX_JIT_BACKGROUND_COMPILE": "0", **subprocess_env()},
             )
 
     def test_compile_after_n_calls(self) -> None:
@@ -348,7 +348,7 @@ def dec(x):
             subprocess.run(
                 [sys.executable, str(test_file)],
                 check=True,
-                env=subprocess_env(),
+                env={"CINDERX_JIT_BACKGROUND_COMPILE": "0", **subprocess_env()},
             )
 
     def test_compile_after_n_calls_predefined(self) -> None:
@@ -381,7 +381,7 @@ def predefined(x):
             subprocess.run(
                 [sys.executable, str(test_file)],
                 check=True,
-                env=subprocess_env(),
+                env={"CINDERX_JIT_BACKGROUND_COMPILE": "0", **subprocess_env()},
             )
 
 

cinderx/PythonLib/test_cinderx/test_jit_preload.py

@@ -60,7 +60,11 @@ def test_func_destroyed_during_preload_multiprocessing(self) -> None:
             cwd=os.path.dirname(__file__),
             capture_output=True,
             encoding=ENCODING,
-            env={**subprocess_env(), "DISABLE_LAZY_IMPORTS": "1"},
+            env={
+                **subprocess_env(),
+                "DISABLE_LAZY_IMPORTS": "1",
+                "CINDERX_JIT_BACKGROUND_COMPILE": "0",
+            },
         )
         self.assertEqual(proc.returncode, 0, proc.stderr)
         self.assertIn("ok ", proc.stdout, proc.stdout)
@@ -88,7 +92,11 @@ def test_func_destroyed_during_preload(self) -> None:
             # DISABLE_LAZY_IMPORTS prevents the safer_lazy_imports startup
             # function from overriding -L with selective lazy imports, which
             # would make the helper modules' imports eager and break this test.
-            env={**subprocess_env(), "DISABLE_LAZY_IMPORTS": "1"},
+            env={
+                **subprocess_env(),
+                "DISABLE_LAZY_IMPORTS": "1",
+                "CINDERX_JIT_BACKGROUND_COMPILE": "0",
+            },
         )
         self.assertEqual(proc.returncode, 0)
         expected_stdout = """resolving a_func

cinderx/PythonLib/test_cinderx/test_jitlist.py

@@ -33,6 +33,13 @@
     "Expecting functions to compile on first call",
 )
 class JitListTest(unittest.TestCase):
+    def setUp(self):
+        self.bg_compile = cinderx.jit.get_background_compile()
+        cinderx.jit.background_compile(False)
+
+    def tearDown(self):
+        cinderx.jit.background_compile(self.bg_compile)
+
     def test_comments(self) -> None:
         cinderx.jit.append_jit_list("")
         initial_jit_list = cinderx.jit.get_jit_list()

cinderx/TestScripts/test_hir_stats.sh

@@ -25,7 +25,7 @@ fi
 
 function run_workload() {
     # Run through cinder_test_runner to skip tests known to break in JIT
-    PYTHONJITDUMPHIRSTATS=1 PYTHONJITALL=1 PYTHONJITDEBUG=1 \
+    CINDERX_JIT_BACKGROUND_COMPILE=0 PYTHONJITDUMPHIRSTATS=1 PYTHONJITALL=1 PYTHONJITDEBUG=1 \
         buck run @//mode/opt "fbcode//cinderx:python$PYTHON_VERSION" -- cinder_test_runner.py test "$@"  -- --randseed=1
 }
 

cinderx/TestScripts/test_multithreaded_compile.sh

@@ -9,7 +9,7 @@ cd "$(dirname "$(readlink -f "$0")")"/../PythonLib
 
 function run() {
     local mode=$1
-    buck2 run @fbcode//mode/"${mode}" fbcode//cinderx:python3.12 -- -X cinderx-jit-all -X cinderx-jit-multithreaded-compile-test -X cinderx-jit-batch-compile-workers="${WORKERS}" -m test_cinderx.multithreaded_compile_test
+    buck2 run @fbcode//mode/"${mode}" fbcode//cinderx:python3.12 -- -X cinderx-jit-background-compile=0 -X cinderx-jit-all -X cinderx-jit-multithreaded-compile-test -X cinderx-jit-batch-compile-workers="${WORKERS}" -m test_cinderx.multithreaded_compile_test
 }
 
 # TASK(T186691817): There should be at least one run with TSAN but we don't have