Open
Description
When a KubeCluster is already in adaptive mode calling cluster.adapt() again results in an error.
cluster.adapt(4, 100)
---------------------------------------------------------------------------
ApiException Traceback (most recent call last)
~/Projects/dask/dask-kubernetes/dask_kubernetes/operator/kubecluster/kubecluster.py in _adapt(self, minimum, maximum)
631 try:
--> 632 await custom_objects_api.patch_namespaced_custom_object_scale(
633 group="kubernetes.dask.org",
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
191 e.body = e.body.decode('utf-8') if six.PY3 else e.body
--> 192 raise e
193
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
184 # perform request and return response
--> 185 response_data = await self.request(
186 method, url, query_params=query_params, headers=header_params,
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in PATCH(self, url, headers, query_params, post_params, body, _preload_content, _request_timeout)
250 _request_timeout=None):
--> 251 return (await self.request("PATCH", url,
252 headers=headers,
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout)
186 if not 200 <= r.status <= 299:
--> 187 raise ApiException(http_resp=r)
188
ApiException: (404)
Reason: Not Found
HTTP response headers: <CIMultiDictProxy('Audit-Id': '80ddf1fd-1751-4fb5-9c04-cdd6891d9ec2', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 12 Dec 2022 16:47:23 GMT', 'Content-Length': '252')>
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"daskautoscalers.kubernetes.dask.org \"simple\" not found","reason":"NotFound","details":{"name":"simple","group":"kubernetes.dask.org","kind":"daskautoscalers"},"code":404}
During handling of the above exception, another exception occurred:
ApiException Traceback (most recent call last)
<ipython-input-10-825575c55b50> in <module>
----> 1 cluster.adapt(4, 100)
~/Projects/dask/dask-kubernetes/dask_kubernetes/operator/kubecluster/kubecluster.py in adapt(self, minimum, maximum)
621 >>> cluster.adapt(minimum=1, maximum=10) # Allow scheduler to add/remove workers within 1-10 range
622 """
--> 623 return self.sync(self._adapt, minimum, maximum)
624
625 async def _adapt(self, minimum=None, maximum=None):
~/Projects/dask/distributed/distributed/utils.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
337 return future
338 else:
--> 339 return sync(
340 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
341 )
~/Projects/dask/distributed/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
404 if error:
405 typ, exc, tb = error
--> 406 raise exc.with_traceback(tb)
407 else:
408 return result
~/Projects/dask/distributed/distributed/utils.py in f()
377 future = asyncio.wait_for(future, callback_timeout)
378 future = asyncio.ensure_future(future)
--> 379 result = yield future
380 except Exception:
381 error = sys.exc_info()
~/.local/lib/python3.8/site-packages/tornado/gen.py in run(self)
760
761 try:
--> 762 value = future.result()
763 except Exception:
764 exc_info = sys.exc_info()
~/Projects/dask/dask-kubernetes/dask_kubernetes/operator/kubecluster/kubecluster.py in _adapt(self, minimum, maximum)
639 )
640 except kubernetes.client.ApiException:
--> 641 await custom_objects_api.create_namespaced_custom_object(
642 group="kubernetes.dask.org",
643 version="v1",
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
190 except ApiException as e:
191 e.body = e.body.decode('utf-8') if six.PY3 else e.body
--> 192 raise e
193
194 self.last_response = response_data
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
183 try:
184 # perform request and return response
--> 185 response_data = await self.request(
186 method, url, query_params=query_params, headers=header_params,
187 post_params=post_params, body=body,
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in POST(self, url, headers, query_params, post_params, body, _preload_content, _request_timeout)
228 post_params=None, body=None, _preload_content=True,
229 _request_timeout=None):
--> 230 return (await self.request("POST", url,
231 headers=headers,
232 query_params=query_params,
~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout)
185
186 if not 200 <= r.status <= 299:
--> 187 raise ApiException(http_resp=r)
188
189 return r
ApiException: (409)
Reason: Conflict
HTTP response headers: <CIMultiDictProxy('Audit-Id': 'fa6d8903-143f-459e-a5aa-303977419144', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Date': 'Mon, 12 Dec 2022 16:47:23 GMT', 'Content-Length': '262')>
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"daskautoscalers.kubernetes.dask.org \"simple\" already exists","reason":"AlreadyExists","details":{"name":"simple","group":"kubernetes.dask.org","kind":"daskautoscalers"},"code":409}I would expect this to patch the existing autoscaler.