Opentelemetry - set status, moved counter, retry_attempts counter

.github/workflows/rust.yml

@@ -111,7 +111,8 @@ jobs:
 
             - name: Run glide-core telemetry tests
               working-directory: ./glide-core/telemetry
-              run: cargo test --all-features
+              run: cargo test --all-features -- --test-threads=1
+              # TODO: Remove `--test-threads=1` once https://github.com/valkey-io/valkey-glide/issues/4057 is resolved
 
             - name: Run glide-ffi tests
               working-directory: ./ffi

CHANGELOG.md

@@ -13,6 +13,7 @@
 * Core/Node: Make OpenTelemetry config to be global per process ([#3771](https://github.com/valkey-io/valkey-glide/pull/3771))
 * Core/Node: Create openTelemetry span to measure command latency ([#3391](https://github.com/valkey-io/valkey-glide/pull/3391))
 * Core: Add `opentelemetry` metrics support ([#3466](https://github.com/valkey-io/valkey-glide/pull/3466))
+* Core: Add `opentelemetry` moved, retry attemps counters and set_status ([#3944](https://github.com/valkey-io/valkey-glide/pull/3944))
 * Node: Fix ZADD, enabling `+inf` and `-inf` as score ([#3370](https://github.com/valkey-io/valkey-glide/pull/3370))
 * Go: Add JSON.SET and JSON.GET ([#3115](https://github.com/valkey-io/valkey-glide/pull/3115))
 * Csharp: updating xUnit, adding xUnit analyser rules and guidelines ([#3035](https://github.com/valkey-io/valkey-glide/pull/3035))

glide-core/redis-rs/redis/Cargo.toml

@@ -96,6 +96,8 @@ uuid = { version = "1.6", optional = true }
 
 telemetrylib = { path = "../../telemetry" }
 
+logger_core = { path = "../../../logger_core" } 
+
 lazy_static = "1"
 
 [features]
@@ -189,7 +191,7 @@ required-features = ["json", "serde/derive"]
 
 [[test]]
 name = "test_cluster_async"
-required-features = ["cluster-async", "tokio-comp"]
+required-features = ["cluster-async", "tokio-comp", "json"]
 
 [[test]]
 name = "test_async_cluster_connections_logic"

glide-core/redis-rs/redis/src/cluster_async/mod.rs

@@ -48,6 +48,9 @@ use pipeline_routing::{
     collect_and_send_pending_requests, map_pipeline_to_nodes, process_and_retry_pipeline_responses,
     route_for_pipeline, PipelineResponses, ResponsePoliciesMap,
 };
+
+use logger_core::log_error;
+
 use std::{
     collections::{HashMap, HashSet},
     fmt, io, mem,
@@ -66,7 +69,7 @@ use tokio::task::JoinHandle;
 
 #[cfg(feature = "tokio-comp")]
 use crate::aio::DisconnectNotifier;
-use telemetrylib::Telemetry;
+use telemetrylib::{GlideOpenTelemetry, Telemetry};
 
 use crate::{
     aio::{get_socket_addrs, ConnectionLike, MultiplexedConnection, Runtime},
@@ -952,6 +955,13 @@ impl<C> Future for Request<C> {
                     return next;
                 }
                 request.retry = request.retry.saturating_add(1);
+                // Record retry attempts metric if telemetry is initialized
+                if let Err(e) = GlideOpenTelemetry::record_retry_attempt() {
+                    log_error(
+                        "OpenTelemetry:retry_error",
+                        format!("Failed to record retry attempt: {}", e),
+                    );
+                }
 
                 if err.kind() == ErrorKind::AllConnectionsUnavailable {
                     return Next::ReconnectToInitialNodes {

glide-core/redis-rs/redis/src/cluster_async/pipeline_routing.rs

@@ -13,10 +13,12 @@ use crate::{cluster_routing, RedisResult, Value};
 use crate::{cluster_routing::Route, Cmd, ErrorKind, RedisError};
 use cluster_routing::RoutingInfo::{MultiNode, SingleNode};
 use futures::FutureExt;
+use logger_core::log_error;
 use rand::prelude::IteratorRandom;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::sync::Arc;
+use telemetrylib::GlideOpenTelemetry;
 use tokio::sync::oneshot;
 use tokio::sync::oneshot::error::RecvError;
 
@@ -95,6 +97,8 @@ pub(crate) type ResponsePoliciesMap =
 /// Adds a command to the pipeline map for a specific node address.
 ///
 /// `add_asking` is a boolean flag that determines whether to add an `ASKING` command before the command.
+/// `is_retrying` is a boolean flag that indicates whether this is a retry attempt.
+#[allow(clippy::too_many_arguments)]
 fn add_command_to_node_pipeline_map<C>(
     pipeline_map: &mut NodePipelineMap<C>,
     address: String,
@@ -103,9 +107,19 @@ fn add_command_to_node_pipeline_map<C>(
     index: usize,
     inner_index: Option<usize>,
     add_asking: bool,
+    is_retrying: bool,
 ) where
     C: Clone,
 {
+    if is_retrying {
+        // Record retry attempt metric if telemetry is initialized
+        if let Err(e) = GlideOpenTelemetry::record_retry_attempt() {
+            log_error(
+                "OpenTelemetry:retry_error",
+                format!("Failed to record retry attempt: {}", e),
+            );
+        }
+    }
     if add_asking {
         let asking_cmd = Arc::new(crate::cmd::cmd("ASKING"));
         pipeline_map
@@ -224,6 +238,7 @@ where
                                     index,
                                     Some(inner_index),
                                     false,
+                                    false,
                                 );
                             }
                         }
@@ -279,7 +294,7 @@ where
         .await
         .map_err(|err| (OperationTarget::NotFound, err))?;
 
-    add_command_to_node_pipeline_map(pipeline_map, address, conn, cmd, index, None, false);
+    add_command_to_node_pipeline_map(pipeline_map, address, conn, cmd, index, None, false, false);
     Ok(())
 }
 
@@ -324,6 +339,7 @@ where
                 index,
                 Some(inner_index),
                 false,
+                false,
             );
         } else {
             return Err((
@@ -1080,6 +1096,7 @@ where
                                 index,
                                 inner_index,
                                 matches!(retry_method, RetryMethod::AskRedirect),
+                                true,
                             );
                             continue;
                         }
@@ -1195,6 +1212,7 @@ where
                     index,
                     inner_index,
                     false,
+                    true,
                 );
             }
             Err(redis_error) => {

glide-core/redis-rs/redis/src/parser.rs

@@ -8,6 +8,9 @@ use crate::types::{
     VerbatimFormat,
 };
 
+use logger_core::log_error;
+use telemetrylib::GlideOpenTelemetry;
+
 use combine::{
     any,
     error::StreamError,
@@ -31,7 +34,16 @@ fn err_parser(line: &str) -> ServerError {
         "EXECABORT" => ServerErrorKind::ExecAbortError,
         "LOADING" => ServerErrorKind::BusyLoadingError,
         "NOSCRIPT" => ServerErrorKind::NoScriptError,
-        "MOVED" => ServerErrorKind::Moved,
+        "MOVED" => {
+            // record moved error metric if telemetry is initialized
+            if let Err(e) = GlideOpenTelemetry::record_moved_error() {
+                log_error(
+                    "OpenTelemetry:moved_error",
+                    format!("Failed to record moved error: {}", e),
+                );
+            }
+            ServerErrorKind::Moved
+        }
         "ASK" => ServerErrorKind::Ask,
         "TRYAGAIN" => ServerErrorKind::TryAgain,
         "CLUSTERDOWN" => ServerErrorKind::ClusterDown,