-
-
Notifications
You must be signed in to change notification settings - Fork 401
Wait for pod resize #1922
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Wait for pod resize #1922
Changes from 4 commits
1b41e26
2149bf6
ba7a08f
4e5d0ae
c386575
3de14b7
786d848
4e98346
e1a7771
4049160
a92af25
f50c8d9
7ad1127
b0a6c6c
c2f30fd
f34eab3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,196 @@ | ||
| use k8s_openapi::api::core::v1::Pod; | ||
| use kube::{ | ||
| Client, | ||
| api::{Api, DeleteParams, Patch, PatchParams, PostParams, ResourceExt}, | ||
| runtime::wait::{await_condition, conditions}, | ||
| }; | ||
| use tracing::*; | ||
|
|
||
| #[tokio::main] | ||
| async fn main() -> anyhow::Result<()> { | ||
| tracing_subscriber::fmt::init(); | ||
| let client = Client::try_default().await?; | ||
|
|
||
| let pods: Api<Pod> = Api::default_namespaced(client); | ||
|
|
||
| // Create a sample pod with resource limits and resize policy | ||
| info!("Creating pod with initial resource requirements"); | ||
| let pod: Pod = serde_json::from_value(serde_json::json!({ | ||
| "apiVersion": "v1", | ||
| "kind": "Pod", | ||
| "metadata": { "name": "resize-wait-demo" }, | ||
| "spec": { | ||
| "containers": [{ | ||
| "name": "app", | ||
| "image": "nginx:1.14.2", | ||
| "resizePolicy": [ | ||
| { | ||
| "resourceName": "cpu", | ||
| "restartPolicy": "NotRequired" | ||
| }, | ||
| { | ||
| "resourceName": "memory", | ||
| "restartPolicy": "RestartContainer" | ||
| } | ||
| ], | ||
| "resources": { | ||
| "requests": { | ||
| "cpu": "100m", | ||
| "memory": "128Mi" | ||
| }, | ||
| "limits": { | ||
| "cpu": "200m", | ||
| "memory": "256Mi" | ||
| } | ||
| } | ||
| }] | ||
| } | ||
| }))?; | ||
|
|
||
| let pp = PostParams::default(); | ||
| match pods.create(&pp, &pod).await { | ||
| Ok(created) => info!("Created pod: {}", created.name_any()), | ||
| Err(kube::Error::Api(ae)) if ae.code == 409 => { | ||
| info!("Pod already exists, deleting and recreating..."); | ||
| pods.delete("resize-wait-demo", &DeleteParams::default()).await?; | ||
| tokio::time::sleep(std::time::Duration::from_secs(5)).await; | ||
| pods.create(&pp, &pod).await?; | ||
| } | ||
| Err(e) => return Err(e.into()), | ||
| } | ||
|
Comment on lines
+65
to
+75
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this can be simplified with server side apply; pseudo code; if the pod exists, no need to delete it, we are overwriting it to have the parameters in the json above |
||
|
|
||
| // Wait for pod to be running | ||
| info!("Waiting for pod to be running..."); | ||
| let running = await_condition(pods.clone(), "resize-wait-demo", conditions::is_pod_running()); | ||
| tokio::time::timeout(std::time::Duration::from_secs(60), running).await??; | ||
| info!("✓ Pod is running"); | ||
|
|
||
| // Display initial resources | ||
| let current = pods.get("resize-wait-demo").await?; | ||
| if let Some(status) = ¤t.status | ||
| && let Some(container_status) = status.container_statuses.as_ref().and_then(|cs| cs.first()) | ||
| { | ||
| info!("Initial container resources: {:?}", container_status.resources); | ||
| info!("Initial resize status: {:?}", status.resize); | ||
| } | ||
|
|
||
| // Resize CPU (no restart required) | ||
| info!("\n--- Example 1: Resizing CPU (NotRequired restart policy) ---"); | ||
| let cpu_patch = serde_json::json!({ | ||
| "spec": { | ||
| "containers": [{ | ||
| "name": "app", | ||
| "resources": { | ||
| "requests": { | ||
| "cpu": "150m" | ||
| }, | ||
| "limits": { | ||
| "cpu": "300m" | ||
| } | ||
| } | ||
| }] | ||
| } | ||
| }); | ||
|
|
||
| let patch_params = PatchParams::default(); | ||
| info!("Patching pod with new CPU resources..."); | ||
| pods.patch_resize("resize-wait-demo", &patch_params, &Patch::Strategic(cpu_patch)) | ||
| .await?; | ||
|
|
||
| info!("Waiting for resize to complete..."); | ||
| let resized = await_condition(pods.clone(), "resize-wait-demo", conditions::is_pod_resized()); | ||
| match tokio::time::timeout(std::time::Duration::from_secs(30), resized).await { | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. let's just |
||
| Ok(Ok(Some(pod))) => { | ||
| info!("✓ Pod resize completed successfully!"); | ||
| if let Some(status) = &pod.status { | ||
| info!("Resize status: {:?}", status.resize); | ||
| if let Some(container_status) = status.container_statuses.as_ref().and_then(|cs| cs.first()) { | ||
| info!( | ||
| "Container resources after CPU resize: {:?}", | ||
| container_status.resources | ||
| ); | ||
| info!("Container restart count: {}", container_status.restart_count); | ||
| } | ||
| } | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. try to factor out some kind of inspector fn that dumps info of a |
||
| } | ||
| Ok(Ok(None)) => warn!("Pod was deleted during resize wait"), | ||
| Ok(Err(e)) => error!("Failed waiting for resize: {}", e), | ||
| Err(_) => { | ||
| warn!("Timeout waiting for resize to complete"); | ||
| let pod = pods.get("resize-wait-demo").await?; | ||
| if let Some(status) = &pod.status { | ||
| warn!("Current resize status: {:?}", status.resize); | ||
| if let Some(conditions) = &status.conditions { | ||
| for cond in conditions { | ||
| if cond.type_ == "PodResizePending" || cond.type_ == "PodResizeInProgress" { | ||
| warn!( | ||
| "Resize condition: type={}, status={}, reason={:?}, message={:?}", | ||
| cond.type_, cond.status, cond.reason, cond.message | ||
| ); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Resize Memory (restart required) | ||
| info!("\n--- Example 2: Resizing Memory (RestartContainer policy) ---"); | ||
| let mem_patch = serde_json::json!({ | ||
| "spec": { | ||
| "containers": [{ | ||
| "name": "app", | ||
| "resources": { | ||
| "requests": { | ||
| "memory": "192Mi" | ||
| }, | ||
| "limits": { | ||
| "memory": "384Mi" | ||
| } | ||
| } | ||
| }] | ||
| } | ||
| }); | ||
|
|
||
| info!("Patching pod with new memory resources..."); | ||
| pods.patch_resize("resize-wait-demo", &patch_params, &Patch::Strategic(mem_patch)) | ||
| .await?; | ||
|
|
||
| info!("Waiting for memory resize to complete (container will restart)..."); | ||
| let resized = await_condition(pods.clone(), "resize-wait-demo", conditions::is_pod_resized()); | ||
| match tokio::time::timeout(std::time::Duration::from_secs(60), resized).await { | ||
| Ok(Ok(Some(pod))) => { | ||
| info!("✓ Pod memory resize completed successfully!"); | ||
| if let Some(status) = &pod.status { | ||
| info!("Resize status: {:?}", status.resize); | ||
| if let Some(container_status) = status.container_statuses.as_ref().and_then(|cs| cs.first()) { | ||
| info!( | ||
| "Container resources after memory resize: {:?}", | ||
| container_status.resources | ||
| ); | ||
| info!( | ||
| "Container restart count: {} (should be >0)", | ||
| container_status.restart_count | ||
| ); | ||
| } | ||
| } | ||
| } | ||
| Ok(Ok(None)) => warn!("Pod was deleted during resize wait"), | ||
| Ok(Err(e)) => error!("Failed waiting for resize: {}", e), | ||
| Err(_) => { | ||
| warn!("Timeout waiting for resize to complete"); | ||
| let pod = pods.get("resize-wait-demo").await?; | ||
| if let Some(status) = &pod.status { | ||
| warn!("Current resize status: {:?}", status.resize); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Cleanup | ||
| info!("\nCleaning up..."); | ||
| let dp = DeleteParams::default(); | ||
| pods.delete("resize-wait-demo", &dp).await?; | ||
| info!("Pod deleted"); | ||
|
|
||
| Ok(()) | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -219,6 +219,25 @@ pub mod conditions { | |
| } | ||
| } | ||
|
|
||
| /// An await condition for `Pod` that returns `true` once an in-place resize operation has completed | ||
| /// | ||
| /// A resize is considered complete when the `status.resize` field is either absent or empty. | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Afaikt this would also rely on the restart having fully started before doing the Also, the docs for the status.resize field suggests that this field is deprecated.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In the end i used only condition ! Are you ok with the implementation ?
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is definitely better, but I do agree that doxx's suggestion from the KEP below would be safer. |
||
| /// This indicates that the kubelet has finished applying the resource changes to the container(s). | ||
| /// | ||
| /// See: <https://kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/> | ||
| #[must_use] | ||
| pub fn is_pod_resized() -> impl Condition<Pod> { | ||
| |obj: Option<&Pod>| { | ||
| if let Some(pod) = obj | ||
| && let Some(status) = &pod.status | ||
| { | ||
| // Resize is complete when the field is absent or empty | ||
| return status.resize.as_ref().is_none_or(String::is_empty); | ||
| } | ||
| false | ||
| } | ||
| } | ||
|
|
||
| /// An await condition for `Job` that returns `true` once it is completed | ||
| #[must_use] | ||
| pub fn is_job_completed() -> impl Condition<Job> { | ||
|
|
@@ -543,6 +562,109 @@ pub mod conditions { | |
| assert!(!is_pod_running().matches_object(None)) | ||
| } | ||
|
|
||
| #[test] | ||
| /// pass when pod resize is complete (resize field is empty) | ||
| fn pod_resized_complete() { | ||
| use super::{Condition, is_pod_resized}; | ||
|
|
||
| let pod = r#" | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: resize-demo | ||
| namespace: default | ||
| spec: | ||
| containers: | ||
| - name: app | ||
| image: nginx:1.14.2 | ||
| status: | ||
| phase: Running | ||
| resize: "" | ||
| "#; | ||
|
|
||
| let p = serde_yaml::from_str(pod).unwrap(); | ||
| assert!(is_pod_resized().matches_object(Some(&p))) | ||
| } | ||
|
|
||
| #[test] | ||
| /// pass when pod resize field is absent | ||
| fn pod_resized_no_field() { | ||
| use super::{Condition, is_pod_resized}; | ||
|
|
||
| let pod = r#" | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: resize-demo | ||
| namespace: default | ||
| spec: | ||
| containers: | ||
| - name: app | ||
| image: nginx:1.14.2 | ||
| status: | ||
| phase: Running | ||
| "#; | ||
|
|
||
| let p = serde_yaml::from_str(pod).unwrap(); | ||
| assert!(is_pod_resized().matches_object(Some(&p))) | ||
| } | ||
|
|
||
| #[test] | ||
| /// fail when pod resize is in progress | ||
| fn pod_resized_in_progress() { | ||
| use super::{Condition, is_pod_resized}; | ||
|
|
||
| let pod = r#" | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: resize-demo | ||
| namespace: default | ||
| spec: | ||
| containers: | ||
| - name: app | ||
| image: nginx:1.14.2 | ||
| status: | ||
| phase: Running | ||
| resize: InProgress | ||
| "#; | ||
|
|
||
| let p = serde_yaml::from_str(pod).unwrap(); | ||
| assert!(!is_pod_resized().matches_object(Some(&p))) | ||
| } | ||
|
|
||
| #[test] | ||
| /// fail when pod resize is proposed | ||
| fn pod_resized_proposed() { | ||
| use super::{Condition, is_pod_resized}; | ||
|
|
||
| let pod = r#" | ||
| apiVersion: v1 | ||
| kind: Pod | ||
| metadata: | ||
| name: resize-demo | ||
| namespace: default | ||
| spec: | ||
| containers: | ||
| - name: app | ||
| image: nginx:1.14.2 | ||
| status: | ||
| phase: Running | ||
| resize: Proposed | ||
| "#; | ||
|
|
||
| let p = serde_yaml::from_str(pod).unwrap(); | ||
| assert!(!is_pod_resized().matches_object(Some(&p))) | ||
| } | ||
|
|
||
| #[test] | ||
| /// fail if pod does not exist | ||
| fn pod_resized_missing() { | ||
| use super::{Condition, is_pod_resized}; | ||
|
|
||
| assert!(!is_pod_resized().matches_object(None)) | ||
| } | ||
|
|
||
| #[test] | ||
| /// pass if job completed | ||
| fn job_completed_ok() { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
7y old docker image. try
stable-alpine