Keep tokio console only in the playground (#3950)

# Description
It turned out that `tokio_unstable` introduces a memory leak when
creating spans. As @MartinquaXD noticed:
> Most of the tracing related memory was allocated while holding a tokio
mutex lock. When you check the tokio code you see that this only happens
when we build it with `tokio_unstable` which is only used for
`tokio-console` support.

Disabling the `tokio_unstable` helped to mitigate the memory leak issue
almost completely.

# Changes

- Remove `tokio_unstable` from the codebase.
- Keep it only in the playground (for now).


## Further implementation
Using the updated deployment CI job from #3954, the tokio-console can be
gated behind a feature in case we need to run it in the future. Will
implement it in a follow-up pr.

Author: squadgazzz

.cargo/config.toml

@@ -10,7 +10,7 @@ axum = { workspace = true, optional = true }
 atty = { workspace = true }
 async-trait = { workspace = true }
 chrono = { workspace = true, features = ["now"] }
-console-subscriber = { workspace = true }
+console-subscriber = { workspace = true, optional = true }
 futures = { workspace = true }
 opentelemetry = { workspace = true }
 opentelemetry-otlp = { workspace = true, features = ["grpc-tonic"] }
@@ -37,3 +37,4 @@ workspace = true
 default = []
 axum-tracing = ["axum"]
 jemalloc-profiling = ["dep:jemalloc_pprof"]
+tokio-console = ["dep:console-subscriber"]

crates/observe/Cargo.toml

@@ -1,4 +1,6 @@
 fn main() {
     // Make build system aware of custom config flags to avoid clippy warnings
+    // tokio_unstable is only used when explicitly compiled with --cfg
+    // tokio_unstable (e.g., in the playground environment)
     println!("cargo::rustc-check-cfg=cfg(tokio_unstable)");
 }

crates/observe/build.rs

@@ -59,19 +59,7 @@ pub fn initialize_reentrant(config: &Config) {
 fn set_tracing_subscriber(config: &Config) {
     let initial_filter = config.env_filter.to_string();
 
-    // The `tracing` APIs are heavily generic to enable zero overhead. Unfortunately
-    // this leads to very annoying type constraints which can only be satisfied
-    // by literally copy and pasting the code so the compiler doesn't try to
-    // infer types that satisfy both the tokio-console and the regular case.
-    // It's tempting to resolve this mess by first configuring the `fmt_layer` and
-    // only then the `console_subscriber`. However, this setup was the only way
-    // I found that:
-    // 1. actually makes `tokio-console` work
-    // 2. prints logs if `tokio-console` is disabled
-    // 3. does NOT skip the next log following a `tracing::event!()`. These calls
-    //    happen for example under the hood in `sqlx`. I don't understand what's
-    //    actually causing that but at this point I'm just happy if all the features
-    //    work correctly.
+    // The `tracing` APIs are heavily generic to enable zero overhead.
 
     macro_rules! fmt_layer {
         ($env_filter:expr_2021, $stderr_threshold:expr_2021, $use_json_format:expr_2021) => {{
@@ -109,11 +97,6 @@ fn set_tracing_subscriber(config: &Config) {
         }};
     }
 
-    let enable_tokio_console: bool = std::env::var("TOKIO_CONSOLE")
-        .unwrap_or("false".to_string())
-        .parse()
-        .unwrap();
-
     let (env_filter, reload_handle) =
         tracing_subscriber::reload::Layer::new(EnvFilter::new(&initial_filter));
 
@@ -161,13 +144,33 @@ fn set_tracing_subscriber(config: &Config) {
         ))
         .with(tracing_layer);
 
-    if cfg!(tokio_unstable) && enable_tokio_console {
-        subscriber.with(console_subscriber::spawn()).init();
-        tracing::info!("started program with support for tokio-console");
-    } else {
+    #[cfg(all(tokio_unstable, feature = "tokio-console"))]
+    {
+        let enable_tokio_console: bool = std::env::var("TOKIO_CONSOLE")
+            .unwrap_or("false".to_string())
+            .parse()
+            .unwrap();
+
+        if enable_tokio_console {
+            subscriber.with(console_subscriber::spawn()).init();
+            tracing::info!("started program with support for tokio-console");
+        } else {
+            subscriber.init();
+            tracing::info!(
+                "started program without support for tokio-console (TOKIO_CONSOLE=false)"
+            );
+        }
+    }
+
+    #[cfg(not(all(tokio_unstable, feature = "tokio-console")))]
+    {
         subscriber.init();
-        tracing::info!("started program without support for tokio-console");
+        tracing::info!(
+            "started program without support for tokio-console (not compiled with tokio_unstable \
+             cfg and tokio-console feature)"
+        );
     }
+
     if cfg!(unix) {
         spawn_reload_handler(initial_filter, reload_handle);
     }

crates/observe/src/tracing.rs

@@ -31,7 +31,7 @@ CMD ["migrate"]
 FROM chef AS builder
 COPY --from=planner /src/recipe.json recipe.json
 COPY --from=chef /.cargo /.cargo
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo chef cook --release --recipe-path recipe.json
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo chef cook --release --features observe/tokio-console --recipe-path recipe.json
 
 # Copy only the library crates for now
 COPY --from=chef /.cargo /.cargo
@@ -48,9 +48,9 @@ COPY ./crates/chain/ ./crates/chain
 COPY ./crates/ethrpc/ ./crates/ethrpc
 COPY ./crates/observe/ ./crates/observe
 COPY ./crates/order-validation/ ./crates/order-validation
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package shared
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package shared
 COPY ./crates/solver/ ./crates/solver
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package solver
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package solver
 
 # Create an base image for all the binaries
 FROM docker.io/debian:bookworm-slim AS base
@@ -61,14 +61,14 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked apt-get update && \
 FROM builder AS alerter-build
 COPY --from=chef /.cargo /.cargo
 COPY ./crates/alerter/ ./crates/alerter
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package alerter
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package alerter
 
 FROM base AS alerter
 COPY --from=alerter-build /src/target/release/alerter /usr/local/bin/alerter
 ENTRYPOINT [ "alerter" ]
 
 FROM builder AS autopilot-build
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package autopilot
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package autopilot
 
 FROM base AS autopilot
 COPY --from=chef /.cargo /.cargo
@@ -78,7 +78,7 @@ ENTRYPOINT [ "autopilot" ]
 
 FROM builder AS driver-build
 COPY ./crates/driver/ ./crates/driver
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package driver
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package driver
 
 FROM base AS driver
 COPY --from=driver-build /src/target/release/driver /usr/local/bin/driver
@@ -87,7 +87,7 @@ ENTRYPOINT [ "driver" ]
 FROM builder AS orderbook-build
 COPY --from=chef /.cargo /.cargo
 COPY ./crates/orderbook/ ./crates/orderbook
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package orderbook
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package orderbook
 
 FROM base AS orderbook
 COPY --from=orderbook-build /src/target/release/orderbook /usr/local/bin/orderbook
@@ -96,7 +96,7 @@ ENTRYPOINT [ "orderbook" ]
 FROM builder AS refunder-build
 COPY --from=chef /.cargo /.cargo
 COPY ./crates/refunder/ ./crates/refunder
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package refunder
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package refunder
 
 FROM base AS refunder
 COPY --from=refunder-build /src/target/release/refunder /usr/local/bin/refunder
@@ -105,7 +105,7 @@ ENTRYPOINT [ "refunder" ]
 FROM builder AS solvers-build
 COPY --from=chef /.cargo /.cargo
 COPY ./crates/solvers/ ./crates/solvers
-RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --package solvers
+RUN CARGO_PROFILE_RELEASE_DEBUG=1 cargo build --release --features observe/tokio-console --package solvers
 
 FROM base AS solvers
 COPY --from=solvers-build /src/target/release/solvers /usr/local/bin/solvers