feat(secret-providers): vault provider (#1450)

* feat: secrets provider module
* feat: Add retry mechanism for Secrets
* feat: Add vault SecretProvider

---------

Co-authored-by: Daniel Orihuela Rodriguez <dorihuela@newrelic.com>
Co-authored-by: danielorihuela <danielorihuela@users.noreply.github.com>

Author: alvarocabanas

Cargo.lock

@@ -2225,6 +2225,12 @@ Distributed under the following license(s):
 
 * MIT
 
+## tokio-tungstenite <https://crates.io/crates/tokio-tungstenite>
+
+Distributed under the following license(s):
+
+* MIT
+
 ## tokio-util <https://crates.io/crates/tokio-util>
 
 Distributed under the following license(s):
@@ -2330,6 +2336,13 @@ Distributed under the following license(s):
 
 * MIT
 
+## tungstenite <https://crates.io/crates/tungstenite>
+
+Distributed under the following license(s):
+
+* MIT
+* Apache-2.0
+
 ## typenum <https://crates.io/crates/typenum>
 
 Distributed under the following license(s):
@@ -2397,6 +2410,13 @@ Distributed under the following license(s):
 * MIT
 * Apache-2.0
 
+## utf-8 <https://crates.io/crates/utf-8>
+
+Distributed under the following license(s):
+
+* MIT
+* Apache-2.0
+
 ## utf8_iter <https://crates.io/crates/utf8_iter>
 
 Distributed under the following license(s):

THIRD_PARTY_NOTICES.md

@@ -46,7 +46,7 @@ wrapper_with_default = { path = "../wrapper_with_default" }
 resource-detection = { path = "../resource-detection" }
 
 # K8S subagent dependencies
-kube = { version = "1.1.0", features = ["runtime", "derive"] }
+kube = { version = "1.1.0", features = ["runtime", "derive", "ws"] }
 k8s-openapi = { version = "0.25.0", features = ["v1_30"] }
 either = { version = "1.15.0" }
 
@@ -76,6 +76,7 @@ opentelemetry-appender-tracing = { version = "0.30.1", features = [
 ] }
 async-trait = "0.1.86"
 
+
 [dev-dependencies]
 assert_cmd = { workspace = true }
 predicates = { workspace = true }
@@ -99,6 +100,7 @@ futures = "0.3.31"
 rcgen = { version = "0.14.0", features = ["crypto"] }
 rustls-pemfile = { version = "2.2.0" }
 rstest = "0.25.0"
+tokio-stream = { version = "0.1.17", features = ["net"] }
 
 [build-dependencies]
 glob = "0.3.2"

agent-control/Cargo.toml

@@ -48,7 +48,7 @@ test/k8s/integration-part1:
 .PHONY: test/k8s/integration-part2
 test/k8s/integration-part2:
 	KUBECONFIG='./tests/k8s/.kubeconfig-dev' minikube update-context
-	tilt ci --file ./tests/k8s/Tiltfile
+	ENABLE_VAULT=true tilt ci --file ./tests/k8s/Tiltfile
 	# reducing the number of threads to 1 forces the tests to run sequentially
 	# We use this approach to split the k8s tests into two parts. We need to update it if this takes too long to run.
 	cargo test k8s_ --  --skip k8s::scenarios --skip k8s::agent_control_cli --nocapture --ignored  --test-threads=1

agent-control/Makefile

@@ -9,6 +9,7 @@ use crate::k8s::client::ClientConfig;
 use crate::opamp::auth::config::AuthConfig;
 use crate::opamp::remote_config::OpampRemoteConfigError;
 use crate::opamp::remote_config::validators::signature::validator::SignatureValidatorConfig;
+use crate::secrets_provider::SecretsProvidersConfig;
 use crate::values::yaml_config::YAMLConfig;
 use crate::{
     agent_type::agent_type_id::AgentTypeID, instrumentation::config::InstrumentationConfig,
@@ -36,9 +37,7 @@ pub struct AgentControlConfig {
     /// fleet_control contains the OpAMP client configuration
     pub fleet_control: Option<OpAMPClientConfig>,
 
-    // We could make this field available only when #[cfg(feature = "k8s")] but it would over-complicate
-    // the struct definition and usage. Making it optional should work no matter what features are enabled.
-    /// k8s is a map containing the kubernetes-specific settings
+    /// kubernetes-specific settings
     #[serde(default)]
     pub k8s: Option<K8sConfig>,
 
@@ -60,6 +59,10 @@ pub struct AgentControlConfig {
     /// A "key-value store" intended to modify agent type definitions, loaded at start time.
     #[serde(default)]
     pub agent_type_var_constraints: VariableConstraints,
+
+    /// configuration for every secrets provider that the current AgentControl instance should be able to access
+    #[serde(default)]
+    pub secrets_providers: Option<SecretsProvidersConfig>,
 }
 
 #[derive(Error, Debug)]

agent-control/src/agent_control/config.rs

@@ -1,5 +1,5 @@
 use crate::agent_control::AgentControl;
-use crate::agent_control::config::K8sConfig;
+use crate::agent_control::config::{AgentControlConfigError, K8sConfig};
 use crate::agent_control::config_repository::repository::AgentControlConfigLoader;
 use crate::agent_control::config_repository::store::AgentControlConfigStore;
 use crate::agent_control::config_validator::RegistryDynamicConfigValidator;
@@ -23,6 +23,7 @@ use crate::opamp::instance_id::k8s::getter::{Identifiers, get_identifiers};
 use crate::opamp::operations::build_opamp_with_channel;
 use crate::opamp::remote_config::validators::SupportedRemoteConfigValidator;
 use crate::opamp::remote_config::validators::regexes::RegexValidator;
+use crate::secrets_provider::SecretsProvidersRegistry;
 use crate::sub_agent::effective_agents_assembler::LocalEffectiveAgentsAssembler;
 use crate::sub_agent::identity::AgentIdentity;
 use crate::sub_agent::k8s::builder::SupervisorBuilderK8s;
@@ -132,10 +133,21 @@ impl AgentControlRunner {
         let template_renderer = TemplateRenderer::default()
             .with_agent_control_variables(agent_control_variables.clone().into_iter());
 
+        let secrets_providers = if let Some(config) = &agent_control_config.secrets_providers {
+            SecretsProvidersRegistry::try_from(config.clone()).map_err(|e| {
+                AgentError::ConfigResolve(AgentControlConfigError::Load(format!(
+                    "Failed to load secrets providers: {e}"
+                )))
+            })?
+        } else {
+            HashMap::default()
+        };
+
         let agents_assembler = Arc::new(LocalEffectiveAgentsAssembler::new(
             self.agent_type_registry.clone(),
             template_renderer,
             self.agent_type_var_constraints,
+            secrets_providers,
         ));
 
         let supervisor_builder =

agent-control/src/agent_control/run/k8s.rs

@@ -1,4 +1,5 @@
 use crate::agent_control::AgentControl;
+use crate::agent_control::config::AgentControlConfigError;
 use crate::agent_control::config_repository::repository::AgentControlConfigLoader;
 use crate::agent_control::config_repository::store::AgentControlConfigStore;
 use crate::agent_control::config_validator::RegistryDynamicConfigValidator;
@@ -23,6 +24,7 @@ use crate::opamp::instance_id::on_host::storer::Storer;
 use crate::opamp::operations::build_opamp_with_channel;
 use crate::opamp::remote_config::validators::SupportedRemoteConfigValidator;
 use crate::opamp::remote_config::validators::regexes::RegexValidator;
+use crate::secrets_provider::SecretsProvidersRegistry;
 use crate::sub_agent::effective_agents_assembler::LocalEffectiveAgentsAssembler;
 use crate::sub_agent::identity::AgentIdentity;
 use crate::sub_agent::on_host::builder::SupervisortBuilderOnHost;
@@ -139,10 +141,21 @@ impl AgentControlRunner {
             )
             .with_agent_control_variables(agent_control_variables.clone().into_iter());
 
+        let secrets_providers = if let Some(config) = &agent_control_config.secrets_providers {
+            SecretsProvidersRegistry::try_from(config.clone()).map_err(|e| {
+                AgentError::ConfigResolve(AgentControlConfigError::Load(format!(
+                    "Failed to load secrets providers: {e}"
+                )))
+            })?
+        } else {
+            HashMap::default()
+        };
+
         let agents_assembler = Arc::new(LocalEffectiveAgentsAssembler::new(
             self.agent_type_registry.clone(),
             template_renderer,
             self.agent_type_var_constraints,
+            secrets_providers,
         ));
 
         let supervisor_builder =

agent-control/src/agent_control/run/on_host.rs

@@ -15,6 +15,7 @@ pub mod http;
 pub mod instrumentation;
 pub mod k8s;
 pub mod opamp;
+pub mod secrets_provider;
 pub mod sub_agent;
 pub mod utils;
 pub mod values;