Reduce logspew from backend simulator (#3369)

Summary:
Pull Request resolved: #3369

What it says on the tin.

Reviewed By: saitcakmak

Differential Revision: D69624873

fbshipit-source-id: 600812a8e63149ea4c7e2c4c03bcd6f1ff2d6f27

ax/benchmark/benchmark.py

@@ -248,6 +248,11 @@ def benchmark_replication(
         seed: The seed to use for this replication.
         strip_runner_before_saving: Whether to strip the runner from the
             experiment before saving it. This enables serialization.
+        scheduler_logging_level: If >INFO, logs will only appear when unexpected
+            things happen. If INFO, logs will update when a trial is completed
+            and when an early stopping strategy, if present, decides whether or
+            not to continue a trial. If DEBUG, logs additionaly include
+            information from a `BackendSimulator`, if present.
 
     Return:
         ``BenchmarkResult`` object.
@@ -263,8 +268,7 @@ def benchmark_replication(
         logging_level=scheduler_logging_level,
     )
     runner = get_benchmark_runner(
-        problem=problem,
-        max_concurrency=scheduler_options.max_pending_trials,
+        problem=problem, max_concurrency=scheduler_options.max_pending_trials
     )
     experiment = Experiment(
         name=f"{problem.name}|{method.name}_{int(time())}",

ax/benchmark/benchmark_runner.py

@@ -184,6 +184,7 @@ def __post_init__(self) -> None:
                     internal_clock=0,
                     use_update_as_start_time=True,
                 ),
+                verbose_logging=False,
             )
             self.simulated_backend_runner = SimulatedBackendRunner(
                 simulator=simulator,

ax/benchmark/tests/test_benchmark.py

@@ -405,6 +405,37 @@ def test_replication_async(self) -> None:
         self._test_replication_async(map_data=False)
         self._test_replication_async(map_data=True)
 
+    def test_logging(self) -> None:
+        method = get_async_benchmark_method()
+        problem = get_async_benchmark_problem(
+            map_data=True,
+        )
+        logger = get_logger("utils.testing.backend_simulator")
+
+        with self.subTest("Logs produced if level is DEBUG"):
+            with self.assertLogs(level=logging.DEBUG, logger=logger):
+                result = benchmark_replication(
+                    problem=problem,
+                    method=method,
+                    seed=0,
+                    strip_runner_before_saving=False,
+                    scheduler_logging_level=logging.DEBUG,
+                )
+            experiment = none_throws(result.experiment)
+            runner = assert_is_instance(experiment.runner, BenchmarkRunner)
+            self.assertFalse(
+                none_throws(runner.simulated_backend_runner).simulator._verbose_logging
+            )
+
+        with self.subTest("Logs not produced by default"), self.assertNoLogs(
+            level=logging.INFO, logger=logger
+        ), self.assertNoLogs(logger=logger):
+            benchmark_replication(
+                problem=problem,
+                method=method,
+                seed=0,
+            )
+
     def test_early_stopping(self) -> None:
         """
         Test early stopping with a deterministic generation strategy and ESS