improved diff with branch main

Author: KimHenrikOtte

candle-core/Cargo.toml

@@ -33,21 +33,15 @@ safetensors = { workspace = true }
 thiserror = { workspace = true }
 yoke = { workspace = true }
 zip = { workspace = true }
-tracing = {workspace = true, optional = true, features = ["release_max_level_off"]}
 
 #Wgpu Dependencies:
-wgpu = { workspace = true, optional = true}
-flume = { workspace = true, optional = true}
+#wgpu = { workspace = true, optional = true }
 bytemuck = { workspace = true, optional = true }
+tracing = {workspace = true, optional = true, features = ["release_max_level_off"]}
+
 pollster = { workspace = true, optional = true }
-log = {workspace = true, optional = true}
-serde_json = { workspace = true, optional = true }
 candle-wgpu-kernels = { workspace = true, optional = true }
 wgpu-compute-layer = { workspace = true, optional = true }
-rustc-hash = {workspace = true, optional = true}
-#for wgpu_debug || wgpu_debug_serialize
-serde = { workspace = true, optional = true }
-
 
 [target.'cfg(all(not(target_arch = "wasm32"), not(target_os = "ios")))'.dependencies]
 candle-ug = { workspace = true, optional = true }
@@ -71,9 +65,23 @@ metal = [
     "candle-ug?/metal",
 ]
 ug = ["dep:candle-ug"]
-wgpu = ["dep:wgpu", "dep:pollster", "dep:bytemuck", "dep:flume", "dep:candle-wgpu-kernels", "dep:log", "dep:tracing", "dep:rustc-hash", "dep:wgpu-compute-layer"]
-wgpu_debug = ["wgpu", "dep:serde", "dep:serde_json", "candle-wgpu-kernels/wgpu_debug_serialize", "wgpu-compute-layer/wgpu_debug"]
-wgpu_debug_serialize = ["wgpu", "dep:serde", "dep:serde_json", "candle-wgpu-kernels/wgpu_debug_serialize", "wgpu-compute-layer/wgpu_debug_serialize"]
+wgpu = [
+    "dep:bytemuck",
+    "dep:tracing",
+    "dep:pollster",
+    "dep:candle-wgpu-kernels",
+    "dep:wgpu-compute-layer",
+]
+wgpu_debug = [
+    "wgpu",
+    "candle-wgpu-kernels/wgpu_debug_serialize",
+    "wgpu-compute-layer/wgpu_debug",
+]
+wgpu_debug_serialize = [
+    "wgpu",
+    "candle-wgpu-kernels/wgpu_debug_serialize",
+    "wgpu-compute-layer/wgpu_debug_serialize",
+]
 
 [[bench]]
 name = "bench_main"

candle-core/benches/benchmarks/matmul_quantized.rs

@@ -84,7 +84,6 @@ fn test_matmul(
                 use candle_core::wgpu::wgpu_functions::matmul::sgemm::GenericDynamicMatmulShaderSettings;
 
                 let mut run_bench = |func_name : String|{
-                    tracing::info!("TEST: {func_name}");
                     if multiple_sizes {
                         group.bench_with_input(
                             BenchmarkId::new(func_name.clone(), size),

candle-core/benches/benchmarks/matmul_wgpu.rs

@@ -86,14 +86,12 @@ fn test_matmul(
                         if tpa { "_tranposedA" } else { "" },
                         if tpb { "_tranposedB" } else { "" }
                     ));
-                    tracing::info!("TEST: {func_name}");
                     if multiple_sizes {
                         group.bench_with_input(
                             BenchmarkId::new(func_name.clone(), size),
                             &size,
                             |b, _| {
                                 b.iter_custom(|iters| {
-                                    tracing::info!("TEST_CUSTOM_ITER: {func_name}");
                                     let start = Instant::now();
                                     for _ in 0..iters {
                                         run(black_box(&lhs), black_box(&rhs));

candle-core/src/wgpu_backend/device.rs

@@ -26,7 +26,7 @@ pub struct WgpuDevice {
 impl WgpuDevice {
     #[instrument]
     pub(crate) async fn create(
-        index: usize,
+        _index: usize,
         configuration: crate::WgpuDeviceConfig,
     ) -> crate::Result<Self> {
         let device = wgpu_compute_layer::WgpuDevice::create_async(configuration.into()).await?;

candle-core/tests/convert_tests.rs

@@ -156,38 +156,3 @@ fn strided_blocks() -> Result<()> {
     };
     Ok(())
 }
-
-fn layout(device: &Device) -> Result<()> {
-    let rs: usize = 14;
-
-    let a: usize = 12;
-    let b: usize = 13;
-
-    let data1 = Tensor::ones((1, b, a, rs), candle_core::DType::U32, &Device::Cpu)?;
-    let data1 = data1.reshape((1, b, a, rs))?;
-    let data2 = data1.to_device(device)?;
-
-    let index1 = data1.i((.., .., 3..6, ..4))?;
-    let index2 = data2.i((.., .., 3..6, ..4))?;
-
-    let result1 = index1.reshape((b, 3, 4))?;
-    let result2 = index2.reshape((b, 3, 4))?;
-
-    assert_eq!(result1.to_vec3::<u32>()?, result2.to_vec3::<u32>()?);
-
-    let copy1 = index1.copy()?;
-    let copy2 = index2.copy()?;
-
-    let result1 = copy1.reshape((b, 3, 4))?;
-    let result2 = copy2.reshape((b, 3, 4))?;
-
-    assert_eq!(result1.to_vec3::<u32>()?, result2.to_vec3::<u32>()?);
-
-    let result1 = index1.sum_all()?.to_vec0::<u32>()?;
-    let result2 = index2.sum_all()?.to_vec0::<u32>()?;
-
-    assert_eq!(result1, result2);
-
-    Ok(())
-}
-test_device!(layout, layout_cpu, layout_gpu, layout_metal, layout_wgpu);

candle-core/tests/layout_tests.rs

@@ -56,18 +56,26 @@ fn avg_pool2d_pytorch(dev: &Device) -> Result<()> {
         dev,
     )?
     .reshape((1, 2, 4, 4))?;
+    let pool = t.avg_pool2d(2)?.squeeze(0)?;
+
     if !dev.is_wgpu() {
-        //-0.16055 rounds to -0.1605 for wgpu
-        let pool = t.avg_pool2d(2)?.squeeze(0)?;
         assert_eq!(
             test_utils::to_vec3_round(&pool, 4)?,
             [
                 [[-1.1926, -0.0395], [0.2688, 0.1871]],
                 [[0.1835, -0.1606], [0.6249, 0.3217]]
             ]
         );
+    } else {
+        //-0.16055 rounds to -0.1605 for wgpu
+        assert_eq!(
+            test_utils::to_vec3_round(&pool, 4)?,
+            [
+                [[-1.1926, -0.0395], [0.2688, 0.1871]],
+                [[0.1835, -0.1605], [0.6249, 0.3217]]
+            ]
+        );
     }
-
     let pool = t.avg_pool2d(3)?.squeeze(0)?;
     assert_eq!(
         test_utils::to_vec3_round(&pool, 4)?,
@@ -105,14 +113,6 @@ fn upsample_nearest2d(dev: &Device) -> Result<()> {
     Ok(())
 }
 
-fn upsample_nearest1d(dev: &Device) -> Result<()> {
-    let t = Tensor::arange(0f32, 3f32, dev)?.reshape((1, 1, 3))?;
-    let upsampled = t.upsample_nearest1d(6)?.i(0)?.i(0)?;
-    assert_eq!(t.i(0)?.i(0)?.to_vec1::<f32>()?, [0.0, 1.0, 2.0]);
-    assert_eq!(upsampled.to_vec1::<f32>()?, [0.0, 0.0, 1.0, 1.0, 2.0, 2.0],);
-    Ok(())
-}
-
 test_device!(
     avg_pool2d,
     avg_pool2d_cpu,
@@ -134,15 +134,6 @@ test_device!(
     max_pool2d_metal,
     max_pool2d_wgpu
 );
-
-test_device!(
-    upsample_nearest1d,
-    upsample_nearest1d_cpu,
-    upsample_nearest1d_gpu,
-    upsample_nearest1d_metal,
-    upsample_nearest1d_wgpu
-);
-
 test_device!(
     upsample_nearest2d,
     upsample_nearest2d_cpu,

candle-core/tests/pool_tests.rs

@@ -64,6 +64,7 @@ bindgen_cuda = { version = "0.1.5", optional = true }
 hf-hub = { workspace = true, features = ["tokio"] }
 
 [features]
+default = []
 accelerate = [
     "dep:accelerate-src",
     "candle/accelerate",
@@ -96,7 +97,6 @@ wgpu_debug = ["wgpu", "candle/wgpu_debug"]
 depth_anything_v2 = ["palette", "enterpolation"]
 tekken = ["tekken-rs"]
 buildtime-download = []
-default = []
 
 [[example]]
 name = "llama_multiprocess"

candle-examples/Cargo.toml

@@ -4,8 +4,6 @@ extern crate intel_mkl_src;
 #[cfg(feature = "accelerate")]
 extern crate accelerate_src;
 
-use std::time::Instant;
-
 use anyhow::Error as E;
 use clap::Parser;
 
@@ -82,7 +80,6 @@ pub fn main() -> anyhow::Result<()> {
     let tokenizer = get_tokenizer(args.tokenizer)?;
     let config = clip::ClipConfig::vit_base_patch32();
     let device = candle_examples::device(args.cpu)?;
-    let start = Instant::now();
     let vec_imgs = match args.images {
         Some(imgs) => imgs,
         None => vec![
@@ -114,7 +111,6 @@ pub fn main() -> anyhow::Result<()> {
             println!("Probability: {:.4}% Text: {} ", p, vec_seq[i]);
         }
     }
-    println!("Total Duration: {:?}", Instant::now().duration_since(start));
     Ok(())
 }
 

candle-examples/examples/clip/main.rs

@@ -369,17 +369,6 @@ fn run_inference(args: &InferenceCmd, common_args: &Args) -> Result<()> {
         print!("{rest}");
     }
     let dt = start_gen.elapsed();
-
-    #[cfg(feature = "wgpu_debug")]
-    {
-        device
-            .as_wgpu_device()
-            .unwrap().inner_device()
-            .log_debuginfo_to_file("", "llama2c", "after_fix_shaders")?;
-        // Example:
-        // log_debuginfo_to_file("", "llama2c", "5.0")?;
-    }
-
     println!(
         "\n{} tokens generated ({:.2} token/s)\n",
         tokens.len(),