refactor(hydro_cli)!: use direct &dyn Any upcasting for Rust 1.86, update pyo3, fix #1821

.github/workflows/ci.yml

@@ -193,6 +193,8 @@ jobs:
 
       - name: Run Python tests (Linux)
         if: ${{ matrix.os == 'ubuntu-latest' }}
+        env:
+          RUST_BACKTRACE: 1
         run: |
           cd hydro_deploy/hydro_cli
           source .venv/bin/activate
@@ -202,6 +204,8 @@ jobs:
 
       - name: Run Python tests (Windows)
         if: ${{ matrix.os == 'windows-latest' }}
+        env:
+          RUST_BACKTRACE: 1
         run: |
           cd hydro_deploy/hydro_cli
           .venv/Scripts/activate

dfir_rs/Cargo.toml

@@ -44,7 +44,7 @@ hydro_deploy_integration = { optional = true, path = "../hydro_deploy/hydro_depl
 itertools = "0.13.0"
 lattices = { path = "../lattices", version = "^0.6.1", features = [ "serde" ] }
 pusherator = { path = "../pusherator", version = "^0.0.12" }
-pyo3 = { optional = true, version = "0.20.0" }
+pyo3 = { optional = true, version = "0.24.0" }
 ref-cast = "1.0.0"
 regex = "1.10.4"
 rustc-hash = "1.1.0"

dfir_rs/src/scheduled/graph.rs

@@ -65,11 +65,8 @@ impl Dfir<'_> {
         let tee_root_data_name = tee_root_data.name.clone();
 
         // Insert new handoff output.
-        let teeing_handoff = tee_root_data
-            .handoff
-            .any_ref()
-            .downcast_ref::<TeeingHandoff<T>>()
-            .unwrap();
+        let teeing_handoff =
+            <dyn Any>::downcast_ref::<TeeingHandoff<T>>(&*tee_root_data.handoff).unwrap();
         let new_handoff = teeing_handoff.tee();
 
         // Handoff ID of new tee output.
@@ -111,11 +108,7 @@ impl Dfir<'_> {
         T: Clone,
     {
         let data = &self.handoffs[tee_port.handoff_id];
-        let teeing_handoff = data
-            .handoff
-            .any_ref()
-            .downcast_ref::<TeeingHandoff<T>>()
-            .unwrap();
+        let teeing_handoff = <dyn Any>::downcast_ref::<TeeingHandoff<T>>(&*data.handoff).unwrap();
         teeing_handoff.drop();
 
         let tee_root = data.pred_handoffs[0];
@@ -875,10 +868,7 @@ impl<'a> Dfir<'a> {
                         .map(|hid| hid.handoff_id)
                         .map(|hid| handoffs.get(hid).unwrap())
                         .map(|h_data| {
-                            h_data
-                                .handoff
-                                .any_ref()
-                                .downcast_ref()
+                            <dyn Any>::downcast_ref(&*h_data.handoff)
                                 .expect("Attempted to cast handoff to wrong type.")
                         })
                         .map(RefCast::ref_cast)
@@ -889,10 +879,7 @@ impl<'a> Dfir<'a> {
                         .map(|hid| hid.handoff_id)
                         .map(|hid| handoffs.get(hid).unwrap())
                         .map(|h_data| {
-                            h_data
-                                .handoff
-                                .any_ref()
-                                .downcast_ref()
+                            <dyn Any>::downcast_ref(&*h_data.handoff)
                                 .expect("Attempted to cast handoff to wrong type.")
                         })
                         .map(RefCast::ref_cast)

dfir_rs/src/scheduled/handoff/handoff_list.rs

@@ -1,5 +1,7 @@
 //! Module for variadic handoff port lists, [`PortList`].
 
+use std::any::Any;
+
 use ref_cast::RefCast;
 use sealed::sealed;
 use variadics::{Variadic, variadic_trait};
@@ -95,13 +97,13 @@ where
     type Ctx<'a> = (&'a PortCtx<S, H>, Rest::Ctx<'a>);
     unsafe fn make_ctx<'a>(&self, handoffs: &'a SlotVec<HandoffTag, HandoffData>) -> Self::Ctx<'a> {
         let (this, rest) = self;
-        let hoff_any = handoffs.get(this.handoff_id).unwrap().handoff.any_ref();
+        let hoff_any: &dyn Any = &*handoffs.get(this.handoff_id).unwrap().handoff;
         debug_assert!(hoff_any.is::<H>());
 
         let handoff = unsafe {
             // SAFETY: Caller must ensure `self` is from `handoffs`.
             // TODO(shadaj): replace with `downcast_ref_unchecked` when it's stabilized
-            &*(hoff_any as *const dyn std::any::Any as *const H)
+            &*(hoff_any as *const dyn Any as *const H)
         };
 
         let ctx = RefCast::ref_cast(handoff);
@@ -117,7 +119,7 @@ where
         let Some(hoff_data) = handoffs.get(this.handoff_id) else {
             panic!("Handoff ID {} not found in `handoffs`.", this.handoff_id);
         };
-        let hoff_any = hoff_data.handoff.any_ref();
+        let hoff_any: &dyn Any = &*hoff_data.handoff;
         assert!(
             hoff_any.is::<H>(),
             "Handoff ID {} is not of type {} in `handoffs`.",

dfir_rs/src/scheduled/handoff/mod.rs

@@ -27,12 +27,6 @@ pub trait CanReceive<T> {
 
 /// A handle onto the metadata part of a [Handoff], with no element type.
 pub trait HandoffMeta: Any {
-    /// Helper to cast an instance of `HandoffMeta` to [`Any`]. In general you cannot cast between
-    /// traits, including [`Any`], but this helper method works around that limitation.
-    ///
-    /// For implementors: the body of this method will generally just be `{ self }`.
-    fn any_ref(&self) -> &dyn Any;
-
     // TODO(justin): more fine-grained info here.
     /// Return if the handoff is empty.
     fn is_bottom(&self) -> bool;
@@ -42,10 +36,6 @@ impl<H> HandoffMeta for Rc<RefCell<H>>
 where
     H: HandoffMeta,
 {
-    fn any_ref(&self) -> &dyn Any {
-        self
-    }
-
     fn is_bottom(&self) -> bool {
         self.borrow().is_bottom()
     }

dfir_rs/src/scheduled/handoff/tee.rs

@@ -1,6 +1,5 @@
 //! Module for teeing handoffs, not currently used much.
 
-use std::any::Any;
 use std::cell::RefCell;
 use std::collections::VecDeque;
 use std::rc::Rc;
@@ -71,10 +70,6 @@ where
 }
 
 impl<T> HandoffMeta for TeeingHandoff<T> {
-    fn any_ref(&self) -> &dyn Any {
-        self
-    }
-
     /// If this output's buffer is empty, return true.
     fn is_bottom(&self) -> bool {
         self.internal.borrow().readers[self.read_from]

dfir_rs/src/scheduled/handoff/vector.rs

@@ -1,4 +1,3 @@
-use std::any::Any;
 use std::cell::{RefCell, RefMut};
 use std::rc::Rc;
 
@@ -65,10 +64,6 @@ impl<T> CanReceive<Vec<T>> for VecHandoff<T> {
 }
 
 impl<T> HandoffMeta for VecHandoff<T> {
-    fn any_ref(&self) -> &dyn Any {
-        self
-    }
-
     fn is_bottom(&self) -> bool {
         (*self.input).borrow_mut().is_empty()
     }

hydro_deploy/core/src/azure.rs

@@ -1,3 +1,4 @@
+use std::any::Any;
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex, OnceLock};
 
@@ -125,10 +126,6 @@ impl Host for AzureHost {
         self.id
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn collect_resources(&self, resource_batch: &mut ResourceBatch) {
         if self.launched.get().is_some() {
             return;
@@ -474,7 +471,7 @@ impl Host for AzureHost {
                 }
             }
             ClientStrategy::InternalTcpPort(target_host) => {
-                if let Some(provider_target) = target_host.as_any().downcast_ref::<AzureHost>() {
+                if let Some(provider_target) = <dyn Any>::downcast_ref::<AzureHost>(target_host) {
                     self.project == provider_target.project
                 } else {
                     false

hydro_deploy/core/src/custom_service.rs

@@ -1,4 +1,4 @@
-use std::any::Any;
+use std::any::TypeId;
 use std::ops::Deref;
 use std::sync::{Arc, OnceLock, Weak};
 
@@ -136,10 +136,6 @@ impl RustCrateSource for CustomClientPort {
 }
 
 impl RustCrateSink for CustomClientPort {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
     fn instantiate(&self, _client_path: &SourcePath) -> Result<Box<dyn FnOnce() -> ServerConfig>> {
         bail!("Custom services cannot be used as the server")
     }
@@ -160,10 +156,15 @@ impl RustCrateSink for CustomClientPort {
         let client_port = wrap_client_port(ServerConfig::from_strategy(&conn_type, server_sink));
 
         Ok(Box::new(move |me| {
+            assert_eq!(
+                me.type_id(),
+                TypeId::of::<CustomClientPort>(),
+                "`instantiate_reverse` received different type than expected."
+            );
             me.downcast_ref::<CustomClientPort>()
                 .unwrap()
                 .record_server_config(client_port);
-            bind_type(server_host.as_any())
+            bind_type(&*server_host)
         }))
     }
 }

hydro_deploy/core/src/gcp.rs

@@ -1,3 +1,4 @@
+use std::any::Any;
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex, OnceLock};
 
@@ -250,10 +251,6 @@ impl Host for GcpComputeEngineHost {
         self.id
     }
 
-    fn as_any(&self) -> &dyn std::any::Any {
-        self
-    }
-
     fn collect_resources(&self, resource_batch: &mut ResourceBatch) {
         if self.launched.get().is_some() {
             return;
@@ -508,7 +505,7 @@ impl Host for GcpComputeEngineHost {
             }
             ClientStrategy::InternalTcpPort(target_host) => {
                 if let Some(gcp_target) =
-                    target_host.as_any().downcast_ref::<GcpComputeEngineHost>()
+                    <dyn Any>::downcast_ref::<GcpComputeEngineHost>(target_host)
                 {
                     self.project == gcp_target.project
                 } else {