Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions examples/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,10 @@ path = "pod_portforward_bind.rs"
name = "pod_resize"
path = "pod_resize.rs"

[[example]]
name = "pod_resize_wait"
path = "pod_resize_wait.rs"

[[example]]
name = "pod_reflector"
path = "pod_reflector.rs"
Expand Down
196 changes: 196 additions & 0 deletions examples/pod_resize_wait.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
use k8s_openapi::api::core::v1::Pod;
use kube::{
Client,
api::{Api, DeleteParams, Patch, PatchParams, PostParams, ResourceExt},
runtime::wait::{await_condition, conditions},
};
use tracing::*;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt::init();
let client = Client::try_default().await?;

let pods: Api<Pod> = Api::default_namespaced(client);

// Create a sample pod with resource limits and resize policy
info!("Creating pod with initial resource requirements");
let pod: Pod = serde_json::from_value(serde_json::json!({
"apiVersion": "v1",
"kind": "Pod",
"metadata": { "name": "resize-wait-demo" },
"spec": {
"containers": [{
"name": "app",
"image": "nginx:1.14.2",
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

7y old docker image. try stable-alpine

"resizePolicy": [
{
"resourceName": "cpu",
"restartPolicy": "NotRequired"
},
{
"resourceName": "memory",
"restartPolicy": "RestartContainer"
}
],
"resources": {
"requests": {
"cpu": "100m",
"memory": "128Mi"
},
"limits": {
"cpu": "200m",
"memory": "256Mi"
}
}
}]
}
}))?;

let pp = PostParams::default();
match pods.create(&pp, &pod).await {
Ok(created) => info!("Created pod: {}", created.name_any()),
Err(kube::Error::Api(ae)) if ae.code == 409 => {
info!("Pod already exists, deleting and recreating...");
pods.delete("resize-wait-demo", &DeleteParams::default()).await?;
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
pods.create(&pp, &pod).await?;
}
Err(e) => return Err(e.into()),
}
Comment on lines +65 to +75
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this can be simplified with server side apply; pseudo code;
let p = pods.patch(pod, serverside).await?;

if the pod exists, no need to delete it, we are overwriting it to have the parameters in the json above


// Wait for pod to be running
info!("Waiting for pod to be running...");
let running = await_condition(pods.clone(), "resize-wait-demo", conditions::is_pod_running());
tokio::time::timeout(std::time::Duration::from_secs(60), running).await??;
info!("✓ Pod is running");

// Display initial resources
let current = pods.get("resize-wait-demo").await?;
if let Some(status) = &current.status
&& let Some(container_status) = status.container_statuses.as_ref().and_then(|cs| cs.first())
{
info!("Initial container resources: {:?}", container_status.resources);
info!("Initial resize status: {:?}", status.resize);
}

// Resize CPU (no restart required)
info!("\n--- Example 1: Resizing CPU (NotRequired restart policy) ---");
let cpu_patch = serde_json::json!({
"spec": {
"containers": [{
"name": "app",
"resources": {
"requests": {
"cpu": "150m"
},
"limits": {
"cpu": "300m"
}
}
}]
}
});

let patch_params = PatchParams::default();
info!("Patching pod with new CPU resources...");
pods.patch_resize("resize-wait-demo", &patch_params, &Patch::Strategic(cpu_patch))
.await?;

info!("Waiting for resize to complete...");
let resized = await_condition(pods.clone(), "resize-wait-demo", conditions::is_pod_resized());
match tokio::time::timeout(std::time::Duration::from_secs(30), resized).await {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let's just ? the error herer

Ok(Ok(Some(pod))) => {
info!("✓ Pod resize completed successfully!");
if let Some(status) = &pod.status {
info!("Resize status: {:?}", status.resize);
if let Some(container_status) = status.container_statuses.as_ref().and_then(|cs| cs.first()) {
info!(
"Container resources after CPU resize: {:?}",
container_status.resources
);
info!("Container restart count: {}", container_status.restart_count);
}
}
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

try to factor out some kind of inspector fn that dumps info of a &Pod rather than inlining it in all the match arms

}
Ok(Ok(None)) => warn!("Pod was deleted during resize wait"),
Ok(Err(e)) => error!("Failed waiting for resize: {}", e),
Err(_) => {
warn!("Timeout waiting for resize to complete");
let pod = pods.get("resize-wait-demo").await?;
if let Some(status) = &pod.status {
warn!("Current resize status: {:?}", status.resize);
if let Some(conditions) = &status.conditions {
for cond in conditions {
if cond.type_ == "PodResizePending" || cond.type_ == "PodResizeInProgress" {
warn!(
"Resize condition: type={}, status={}, reason={:?}, message={:?}",
cond.type_, cond.status, cond.reason, cond.message
);
}
}
}
}
}
}

// Resize Memory (restart required)
info!("\n--- Example 2: Resizing Memory (RestartContainer policy) ---");
let mem_patch = serde_json::json!({
"spec": {
"containers": [{
"name": "app",
"resources": {
"requests": {
"memory": "192Mi"
},
"limits": {
"memory": "384Mi"
}
}
}]
}
});

info!("Patching pod with new memory resources...");
pods.patch_resize("resize-wait-demo", &patch_params, &Patch::Strategic(mem_patch))
.await?;

info!("Waiting for memory resize to complete (container will restart)...");
let resized = await_condition(pods.clone(), "resize-wait-demo", conditions::is_pod_resized());
match tokio::time::timeout(std::time::Duration::from_secs(60), resized).await {
Ok(Ok(Some(pod))) => {
info!("✓ Pod memory resize completed successfully!");
if let Some(status) = &pod.status {
info!("Resize status: {:?}", status.resize);
if let Some(container_status) = status.container_statuses.as_ref().and_then(|cs| cs.first()) {
info!(
"Container resources after memory resize: {:?}",
container_status.resources
);
info!(
"Container restart count: {} (should be >0)",
container_status.restart_count
);
}
}
}
Ok(Ok(None)) => warn!("Pod was deleted during resize wait"),
Ok(Err(e)) => error!("Failed waiting for resize: {}", e),
Err(_) => {
warn!("Timeout waiting for resize to complete");
let pod = pods.get("resize-wait-demo").await?;
if let Some(status) = &pod.status {
warn!("Current resize status: {:?}", status.resize);
}
}
}

// Cleanup
info!("\nCleaning up...");
let dp = DeleteParams::default();
pods.delete("resize-wait-demo", &dp).await?;
info!("Pod deleted");

Ok(())
}
122 changes: 122 additions & 0 deletions kube-runtime/src/wait.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,25 @@ pub mod conditions {
}
}

/// An await condition for `Pod` that returns `true` once an in-place resize operation has completed
///
/// A resize is considered complete when the `status.resize` field is either absent or empty.
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Afaikt this would also rely on the restart having fully started before doing the await_condition? Otherwise we might return early if we start waiting at the same time as starting the resize (if we are unlucky).

Also, the docs for the status.resize field suggests that this field is deprecated.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the end i used only condition ! Are you ok with the implementation ?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is definitely better, but I do agree that doxx's suggestion from the KEP below would be safer.
you should have all the data still since the Condition takes a Pod.

/// This indicates that the kubelet has finished applying the resource changes to the container(s).
///
/// See: <https://kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/>
#[must_use]
pub fn is_pod_resized() -> impl Condition<Pod> {
|obj: Option<&Pod>| {
if let Some(pod) = obj
&& let Some(status) = &pod.status
{
// Resize is complete when the field is absent or empty
return status.resize.as_ref().is_none_or(String::is_empty);
}
false
}
}

/// An await condition for `Job` that returns `true` once it is completed
#[must_use]
pub fn is_job_completed() -> impl Condition<Job> {
Expand Down Expand Up @@ -543,6 +562,109 @@ pub mod conditions {
assert!(!is_pod_running().matches_object(None))
}

#[test]
/// pass when pod resize is complete (resize field is empty)
fn pod_resized_complete() {
use super::{Condition, is_pod_resized};

let pod = r#"
apiVersion: v1
kind: Pod
metadata:
name: resize-demo
namespace: default
spec:
containers:
- name: app
image: nginx:1.14.2
status:
phase: Running
resize: ""
"#;

let p = serde_yaml::from_str(pod).unwrap();
assert!(is_pod_resized().matches_object(Some(&p)))
}

#[test]
/// pass when pod resize field is absent
fn pod_resized_no_field() {
use super::{Condition, is_pod_resized};

let pod = r#"
apiVersion: v1
kind: Pod
metadata:
name: resize-demo
namespace: default
spec:
containers:
- name: app
image: nginx:1.14.2
status:
phase: Running
"#;

let p = serde_yaml::from_str(pod).unwrap();
assert!(is_pod_resized().matches_object(Some(&p)))
}

#[test]
/// fail when pod resize is in progress
fn pod_resized_in_progress() {
use super::{Condition, is_pod_resized};

let pod = r#"
apiVersion: v1
kind: Pod
metadata:
name: resize-demo
namespace: default
spec:
containers:
- name: app
image: nginx:1.14.2
status:
phase: Running
resize: InProgress
"#;

let p = serde_yaml::from_str(pod).unwrap();
assert!(!is_pod_resized().matches_object(Some(&p)))
}

#[test]
/// fail when pod resize is proposed
fn pod_resized_proposed() {
use super::{Condition, is_pod_resized};

let pod = r#"
apiVersion: v1
kind: Pod
metadata:
name: resize-demo
namespace: default
spec:
containers:
- name: app
image: nginx:1.14.2
status:
phase: Running
resize: Proposed
"#;

let p = serde_yaml::from_str(pod).unwrap();
assert!(!is_pod_resized().matches_object(Some(&p)))
}

#[test]
/// fail if pod does not exist
fn pod_resized_missing() {
use super::{Condition, is_pod_resized};

assert!(!is_pod_resized().matches_object(None))
}

#[test]
/// pass if job completed
fn job_completed_ok() {
Expand Down