File tree Expand file tree Collapse file tree 2 files changed +5
-3
lines changed
shortfin/python/shortfin_apps/llm/components Expand file tree Collapse file tree 2 files changed +5
-3
lines changed Original file line number Diff line number Diff line change @@ -104,11 +104,11 @@ async def run(self):
104
104
# Decode loop.
105
105
await self .token_selector .decode (self .exec_req )
106
106
finally :
107
- self .exec_req .completed .set_success ()
107
+ self .exec_req .request_exec_success .set_success ()
108
108
self .exec_req .free_cache_pages ()
109
109
110
110
async def await_completion (self ):
111
- await self .exec_req .completed
111
+ await self .exec_req .request_exec_success
112
112
return self .index
113
113
114
114
def results_callback (self , result : int | list [list [int ]]):
Original file line number Diff line number Diff line change @@ -37,7 +37,9 @@ def __init__(
37
37
self .input_token_ids = input_token_ids
38
38
self .prompt_length = len (input_token_ids )
39
39
self .done = sf .VoidFuture ()
40
- self .completed = sf .VoidFuture ()
40
+ # This is set to success once the request has been decoded successfully
41
+ # and response is ready to be sent back to the client.
42
+ self .request_exec_success = sf .VoidFuture ()
41
43
self .rid = rid
42
44
# Unique `instance_id` for token selection strategies that may need
43
45
# to differentiate between an original req and a copy of a req.
You can’t perform that action at this time.
0 commit comments