fix(release): during cleanup let pod deletion 404 in case Kubernetes cleaned up as well (#1005)

Fixes #1002

Author: helgi

rootfs/api/models/release.py

@@ -248,7 +248,7 @@ def delete(self, *args, **kwargs):
         finally:
             super(Release, self).delete(*args, **kwargs)
 
-    def cleanup_old(self):
+    def cleanup_old(self):  # noqa
         """Cleanup all but the latest release from Kubernetes"""
         latest_version = 'v{}'.format(self.version)
         self.app.log(
@@ -307,7 +307,12 @@ def cleanup_old(self):
             if current_version == latest_version:
                 continue
 
-            self._scheduler.delete_pod(self.app.id, pod['metadata']['name'])
+            try:
+                self._scheduler.delete_pod(self.app.id, pod['metadata']['name'])
+            except KubeHTTPException as e:
+                # Sometimes k8s will manage to remove the pod from under us
+                if e.response.status_code == 404:
+                    continue
 
         self._cleanup_deployment_secrets_and_configs(self.app.id)
 

rootfs/scheduler/__init__.py

@@ -153,7 +153,12 @@ def cleanup_release(self, namespace, controller, timeout):
             if self.pod_deleted(pod):
                 continue
 
-            self.delete_pod(namespace, pod['metadata']['name'])
+            try:
+                self.delete_pod(namespace, pod['metadata']['name'])
+            except KubeHTTPException as e:
+                # Sometimes k8s will manage to remove the pod from under us
+                if e.response.status_code == 404:
+                    continue
 
     def _get_deploy_steps(self, batches, tags):
         # if there is no batch information available default to available nodes for app