feat: add environment variable for controlling retry iterations

TianyiQ · TianyiQ · commit ac4ed32200b9 · 2024-12-29T05:15:29.000Z
diff --git a/src/abstractions/backends.py b/src/abstractions/backends.py
@@ -584,7 +584,8 @@ def sglang_process_batch(
             assert len(output) == len(sample_dicts)
 
             count = 0
-            for _ in range(20):
+            max_iter = int(os.environ.get("SG_ITER", 20))
+            for _ in range(max_iter):
                 bad_indices = [
                     k
                     for k in range(len(output))

Original file line number	Diff line number	Diff line change
`@@ -584,7 +584,8 @@ def sglang_process_batch(`
`584`	`584`	`assert len(output) == len(sample_dicts)`
`585`	`585`
`586`	`586`	`count = 0`
`587`		`- for _ in range(20):`
	`587`	`+ max_iter = int(os.environ.get("SG_ITER", 20))`
	`588`	`+ for _ in range(max_iter):`
`588`	`589`	`bad_indices = [`
`589`	`590`	`k`
`590`	`591`	`for k in range(len(output))`