Skip to content

Commit 18a3bed

Browse files
LostRuinsLostRuins
committed
fixed a deadlock
1 parent ee2ecfb commit 18a3bed

2 files changed

Lines changed: 109 additions & 53 deletions

File tree

‎embd_res/klite.embd‎

Lines changed: 69 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -760,6 +760,10 @@ Current version indicated by LITEVER below.
760760
}
761761
}
762762

763+
#chat_msg_body {
764+
padding-bottom: 1.5rem;
765+
}
766+
763767

764768
/* Viewports */
765769
#maineditbody
@@ -4550,6 +4554,7 @@ Current version indicated by LITEVER below.
45504554
adaptivep_target: -1.0,
45514555
adaptivep_decay: 0.9,
45524556
sampler_order: [6, 0, 1, 3, 4, 2, 5],
4557+
custom_sampler_fields: {},
45534558
};
45544559

45554560
const defaultsettings = JSON.parse(JSON.stringify(localsettings));
@@ -12813,6 +12818,30 @@ Current version indicated by LITEVER below.
1281312818
document.getElementById("secondepsamplecontainer").classList.remove("hidden");
1281412819
}
1281512820

12821+
var pendingcustomfieldssampler = {};
12822+
function show_customfields_sampler()
12823+
{
12824+
inputBoxOkCancel("Insert JSON object containing custom fields to send. These values will be added to every generation payload.","Extra Custom Fields",JSON.stringify(pendingcustomfieldssampler),"Paste JSON Here",()=>{
12825+
let userinput = getInputBoxValue().trim();
12826+
try
12827+
{
12828+
pendingcustomfieldssampler = {};
12829+
if(userinput!="")
12830+
{
12831+
pendingcustomfieldssampler = JSON.parse(userinput);
12832+
}
12833+
document.getElementById("custom_field_overview").innerText = (pendingcustomfieldssampler && Object.keys(pendingcustomfieldssampler).length>0)?(`${Object.keys(pendingcustomfieldssampler).length} items`):"OFF";
12834+
sampler_setting_tweaked();
12835+
} catch (e) {
12836+
console.log("Custom fields incorrectly formatted: "+e);
12837+
}
12838+
12839+
},
12840+
()=>{
12841+
//do nothing on cancel
12842+
},false,true);
12843+
}
12844+
1281612845
function explain_horde()
1281712846
{
1281812847
msgbox("The AI Horde generates text using crowdsourced GPUs by volunteer workers. By default your inputs are not logged, but as Horde workers are open source, they can be modified to do so. <br><br>In all cases, the sender will *always be anonymous*, however you are still advised to avoid sending privacy sensitive information.<br>","Disclaimer",true);
@@ -15797,6 +15826,7 @@ Current version indicated by LITEVER below.
1579715826
document.getElementById("nsigma").value = localsettings.nsigma;
1579815827
document.getElementById("dynatemp_overview").innerText = (localsettings.dynatemp_range!=0?"ON":"OFF");
1579915828
document.getElementById("second_ep_overview").innerText = (localsettings.second_ep_qty>0 && localsettings.second_ep_url?"ON":"OFF");
15829+
document.getElementById("custom_field_overview").innerText = (localsettings.custom_sampler_fields && Object.keys(localsettings.custom_sampler_fields).length>0)?(`${Object.keys(localsettings.custom_sampler_fields).length} items`):"OFF";
1580015830
document.getElementById("presence_penalty").value = localsettings.presence_penalty;
1580115831
document.getElementById("sampler_seed").value = localsettings.sampler_seed;
1580215832
document.getElementById("top_k").value = document.getElementById("top_k_slide").value = localsettings.top_k;
@@ -15934,6 +15964,7 @@ Current version indicated by LITEVER below.
1593415964
document.getElementById("imagestyleinput").value = localsettings.image_styles;
1593515965
document.getElementById("negpromptinput").value = localsettings.image_negprompt;
1593615966
pendinggrammar = localsettings.grammar;
15967+
pendingcustomfieldssampler = JSON.parse(JSON.stringify(localsettings.custom_sampler_fields));
1593715968

1593815969
//prepare the input for sampler order
1593915970
let samplerstr = localsettings.sampler_order.toString();
@@ -16138,13 +16169,15 @@ Current version indicated by LITEVER below.
1613816169
document.getElementById("miro_tau").value = found.miro_tau == null? defaultsettings.miro_tau : found.miro_tau;
1613916170
document.getElementById("miro_eta").value = found.miro_eta == null? defaultsettings.miro_eta : found.miro_eta;
1614016171
pendinggrammar = found.grammar == null? defaultsettings.grammar : found.grammar;
16172+
pendingcustomfieldssampler = found.custom_sampler_fields == null? defaultsettings.custom_sampler_fields : found.custom_sampler_fields;
1614116173
document.getElementById("dynatemp_range").value = found.dynatemp_range;
1614216174
document.getElementById("dynatemp_exponent").value = found.dynatemp_exponent;
1614316175
document.getElementById("dynatemp_overview").innerText = (document.getElementById("dynatemp_range").value!=0?"ON":"OFF");
1614416176
document.getElementById("second_ep_qty").value = found.second_ep_qty == null? defaultsettings.second_ep_qty : found.second_ep_qty;
1614516177
document.getElementById("second_ep_model").value = found.second_ep_model == null? defaultsettings.second_ep_model : found.second_ep_model;
1614616178
document.getElementById("second_ep_url").value = found.second_ep_url == null? defaultsettings.second_ep_url : found.second_ep_url;
1614716179
document.getElementById("second_ep_overview").innerText = (document.getElementById("second_ep_qty").value>0 && document.getElementById("second_ep_url").value!=""?"ON":"OFF");
16180+
document.getElementById("custom_field_overview").innerText = (pendingcustomfieldssampler && Object.keys(pendingcustomfieldssampler).length>0)?(`${Object.keys(pendingcustomfieldssampler).length} items`):"OFF";
1614816181
} else {
1614916182
document.getElementById("presetsdesc").innerText = "";
1615016183
}
@@ -16281,6 +16314,7 @@ Current version indicated by LITEVER below.
1628116314
miro_tau: parseFloat(document.getElementById("miro_tau").value),
1628216315
miro_eta: parseFloat(document.getElementById("miro_eta").value),
1628316316
grammar: pendinggrammar,
16317+
custom_sampler_fields: pendingcustomfieldssampler,
1628416318
dynatemp_range: parseFloat(document.getElementById("dynatemp_range").value),
1628516319
dynatemp_exponent: parseFloat(document.getElementById("dynatemp_exponent").value),
1628616320
second_ep_qty: document.getElementById("second_ep_qty").value,
@@ -16427,6 +16461,7 @@ Current version indicated by LITEVER below.
1642716461
document.getElementById("adaptivep_decay").value != (found.adaptivep_decay != null ? found.adaptivep_decay : defaultsettings.adaptivep_decay) ||
1642816462
document.getElementById("nsigma").value != found.nsigma ||
1642916463
pendinggrammar != (found.grammar ? found.grammar : defaultsettings.grammar) ||
16464+
JSON.stringify(pendingcustomfieldssampler) != (found.custom_sampler_fields ? JSON.stringify(found.custom_sampler_fields) : JSON.stringify(defaultsettings.custom_sampler_fields)) ||
1643016465
document.getElementById("dynatemp_range").value != found.dynatemp_range ||
1643116466
document.getElementById("dynatemp_exponent").value != found.dynatemp_exponent ||
1643216467
document.getElementById("second_ep_qty").value != (found.second_ep_qty != null ? found.second_ep_qty : defaultsettings.second_ep_qty) ||
@@ -16847,6 +16882,7 @@ Current version indicated by LITEVER below.
1684716882
localsettings.image_styles = document.getElementById("imagestyleinput").value;
1684816883
localsettings.image_negprompt = document.getElementById("negpromptinput").value;
1684916884
localsettings.grammar = pendinggrammar;
16885+
localsettings.custom_sampler_fields = pendingcustomfieldssampler;
1685016886
localsettings.tokenstreammode = document.getElementById("tokenstreammode").value;
1685116887
localsettings.img_autogen_type = document.getElementById("img_autogen_type").value;
1685216888
localsettings.img_crop = (document.getElementById("img_crop").checked ? true : false);
@@ -17446,6 +17482,8 @@ Current version indicated by LITEVER below.
1744617482
"models": selected_models.map((m) => { return m.name }),
1744717483
};
1744817484

17485+
Object.assign(submit_payload.params, localsettings.custom_sampler_fields);
17486+
1744917487
if (localsettings.sampler_seed >= 1) {
1745017488
submit_payload.params.sampler_seed = localsettings.sampler_seed;
1745117489
}
@@ -17522,6 +17560,8 @@ Current version indicated by LITEVER below.
1752217560
"models": selected_models.map((m) => { return m.name }),
1752317561
};
1752417562

17563+
Object.assign(submit_payload.params, localsettings.custom_sampler_fields);
17564+
1752517565
if(localsettings.sampler_seed>=1)
1752617566
{
1752717567
submit_payload.params.sampler_seed = localsettings.sampler_seed;
@@ -20515,6 +20555,8 @@ Current version indicated by LITEVER below.
2051520555
"models": selected_models.map((m) => { return m.name }),
2051620556
};
2051720557

20558+
Object.assign(submit_payload.params, localsettings.custom_sampler_fields);
20559+
2051820560
if(is_using_kcpp_with_added_memory())
2051920561
{
2052020562
submit_payload.params.memory = truncated_memory;
@@ -21008,6 +21050,16 @@ Current version indicated by LITEVER below.
2100821050
"temperature": submit_payload.params.temperature,
2100921051
"top_p": submit_payload.params.top_p
2101021052
}
21053+
if(document.getElementById("useoainonstandard").checked)
21054+
{
21055+
//send ALL non standard params
21056+
if(submit_payload.params.sampler_seed>=1)
21057+
{
21058+
oai_payload.seed = submit_payload.params.sampler_seed;
21059+
}
21060+
Object.assign(oai_payload, submit_payload.params);
21061+
}
21062+
Object.assign(oai_payload, localsettings.custom_sampler_fields);
2101121063
if(localsettings.request_logprobs && !targetep.toLowerCase().includes("api.x.ai") && !targetep.toLowerCase().includes("api.mistral.ai"))
2101221064
{
2101321065
if(document.getElementById("useoaichatcompl").checked || targetep.toLowerCase().includes("api.x.ai"))
@@ -21034,18 +21086,10 @@ Current version indicated by LITEVER below.
2103421086
if(!targetep.toLowerCase().includes("pollinations.ai") && !targetep.toLowerCase().includes("api.mistral.ai") && !targetep.toLowerCase().includes("api.x.ai"))
2103521087
{
2103621088
//mistral api does not support presence pen
21037-
oai_payload.presence_penalty = scaled_rep_pen;
21038-
}
21039-
if(document.getElementById("useoainonstandard").checked)
21040-
{
21041-
//featherless api supports additional fields, include them
21042-
oai_payload.top_k = (submit_payload.params.top_k<1?300:submit_payload.params.top_k);
21043-
oai_payload.min_p = localsettings.min_p;
21044-
if(submit_payload.params.sampler_seed>=1)
21089+
if(scaled_rep_pen>0)
2104521090
{
21046-
oai_payload.seed = submit_payload.params.sampler_seed;
21091+
oai_payload.presence_penalty = scaled_rep_pen;
2104721092
}
21048-
oai_payload.top_a = localsettings.top_a;
2104921093
}
2105021094
if(submit_payload.params.logit_bias && JSON.stringify(submit_payload.params.logit_bias) != '{}')
2105121095
{
@@ -21312,6 +21356,7 @@ Current version indicated by LITEVER below.
2131221356
"max_tokens": submit_payload.params.max_length,
2131321357
"temperature": submit_payload.params.temperature,
2131421358
};
21359+
Object.assign(claude_payload, localsettings.custom_sampler_fields);
2131521360
claude_payload.messages.push({"role": "user", "content": submit_payload.prompt})
2131621361
if(sysprompt)
2131721362
{
@@ -21528,6 +21573,8 @@ Current version indicated by LITEVER below.
2152821573
}
2152921574
};
2153021575

21576+
Object.assign(payload, localsettings.custom_sampler_fields);
21577+
2153121578
if(document.getElementById("usegeminiweb").checked)
2153221579
{
2153321580
payload["tools"] = [{"google_search": {}}];
@@ -21710,6 +21757,8 @@ Current version indicated by LITEVER below.
2171021757
submit_payload.params.smoothing_factor = localsettings.smoothing_factor;
2171121758
// submit_payload.params.smoothing_curve = localsettings.smoothing_curve; //no idea if horde supports this, dont care to check
2171221759
// submit_payload.params.nsigma = localsettings.nsigma;
21760+
21761+
Object.assign(submit_payload.params, localsettings.custom_sampler_fields);
2171321762
}
2171421763

2171521764
last_request_str = JSON.stringify(submit_payload);
@@ -30130,6 +30179,15 @@ Current version indicated by LITEVER below.
3013030179
</div>
3013130180
</div>
3013230181
</div>
30182+
<div style="display:flex;width:100%;">
30183+
<div class="settinglabel settingcell">
30184+
<div class="justifyleft" style="width:100%">Extra Custom Fields <span class="helpicon">?<span class="helptext">
30185+
Define custom parameters to send along with all generation payloads. Advanced users only.</span></span></div>
30186+
<div class="justifyleft" style="width:100%;">
30187+
<button title="Extra Custom Fields" type="button" class="btn btn-primary" style="padding:2px 4px;font-size:12px;" onclick="show_customfields_sampler()"><span id="custom_field_overview">OFF</span></button>
30188+
</div>
30189+
</div>
30190+
</div>
3013330191
</div>
3013430192
</div>
3013530193
</div>
@@ -31440,7 +31498,7 @@ Current version indicated by LITEVER below.
3144031498
<div><input type="checkbox" id="useoaichatcompl" title="Use ChatCompletions API" onchange="toggleoaichatcompl()">
3144131499
<div class="box-label">Chat-Completions API</div></div>
3144231500
<div><input type="checkbox" id="useoainonstandard" title="Send Non-Standard Fields">
31443-
<div class="box-label">Non-Standard Fields <span class="helpicon">?<span class="helptext">Send extra non-standard samplers like Min-P, Top-K and Top-A.</span></span></div></div>
31501+
<div class="box-label">Non-Standard Fields <span class="helpicon">?<span class="helptext">Send all extra non-standard samplers (e.g Min-P, Top-K, Top-A, TFS). May be rejected by many endpoints.</span></span></div></div>
3144431502
</div>
3144531503
<div id="useoaichatcomplbox" class="hidden" onload="toggleoaichatcompl();">
3144631504
<div style="display: inline-block;">

‎koboldcpp.py‎

Lines changed: 40 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
extra_images_max = 4 # for kontext/qwen img
7474

7575
# global vars
76-
KcppVersion = "1.112.1"
76+
KcppVersion = "1.112.2"
7777
showdebug = True
7878
kcpp_instance = None #global running instance
7979
global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False, "restart_override_base_config":"", "last_active_timestamp":datetime.now(), "triggered_sleeping":False, "current_model":"initial_model", "base_config":"", "swapReqType": None, "autoswapmode": False}
@@ -4401,30 +4401,29 @@ def _handle(self):
44014401

44024402
if is_different_model or was_auto_unloaded:
44034403
model_switch_pass = True
4404-
with proxy_reload_lock:
4405-
whitelist = get_current_admindir_list() # see if its an allowed swap
4406-
if was_auto_unloaded and not model_name:
4407-
model_name = "initial_model"
4408-
if is_different_model and (model_name in whitelist):
4409-
global_memory["last_active_timestamp"] = datetime.now()
4410-
global_memory["triggered_sleeping"] = False
4411-
reqbody = json.dumps({"filename":model_name})
4412-
reqheaders = {
4413-
'Content-Type': 'application/json',
4414-
'Content-Length': str(len(reqbody)),
4415-
}
4416-
if args.adminpassword:
4417-
reqheaders["Authorization"] = f"Bearer {args.adminpassword}"
4418-
conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600)
4419-
conn.request("POST", "/api/admin/reload_config", body=reqbody, headers=reqheaders)
4420-
resp = conn.getresponse()
4421-
time.sleep(3)
4422-
global_memory["last_active_timestamp"] = datetime.now()
4423-
global_memory["triggered_sleeping"] = False
4424-
if not self.wait_for_upstream_ready(upstream_port,120,0.5):
4425-
self.send_error(504, "KoboldCpp model swap reload timed out")
4426-
return
4427-
time.sleep(0.1)
4404+
whitelist = get_current_admindir_list() # see if its an allowed swap
4405+
if was_auto_unloaded and not model_name:
4406+
model_name = "initial_model"
4407+
if is_different_model and (model_name in whitelist):
4408+
global_memory["last_active_timestamp"] = datetime.now()
4409+
global_memory["triggered_sleeping"] = False
4410+
reqbody = json.dumps({"filename":model_name})
4411+
reqheaders = {
4412+
'Content-Type': 'application/json',
4413+
'Content-Length': str(len(reqbody)),
4414+
}
4415+
if args.adminpassword:
4416+
reqheaders["Authorization"] = f"Bearer {args.adminpassword}"
4417+
conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600)
4418+
conn.request("POST", "/api/admin/reload_config", body=reqbody, headers=reqheaders)
4419+
resp = conn.getresponse()
4420+
time.sleep(3)
4421+
global_memory["last_active_timestamp"] = datetime.now()
4422+
global_memory["triggered_sleeping"] = False
4423+
if not self.wait_for_upstream_ready(upstream_port,120,0.5):
4424+
self.send_error(504, "KoboldCpp model swap reload timed out")
4425+
return
4426+
time.sleep(0.1)
44284427
if autoswapEnabled and not model_switch_pass:
44294428
textReqs = ["/api/extra/generate/stream","/api/extra/tokencount","/api/v1/generate","/sdapi/v1/interrogate","/v1/completions","/v1/chat/completions","/v1/responses","/completions","/chat/completions","/responses"]
44304429
sttReqs = ["/api/extra/transcribe","/v1/audio/transcriptions"]
@@ -4454,23 +4453,22 @@ def _handle(self):
44544453
swapModeChanged = True
44554454

44564455
if (global_memory["swapReqType"] is not None and swapModeChanged):
4457-
with proxy_reload_lock:
4458-
reqbody = json.dumps({"filename":global_memory["current_model"], "baseconfig": global_memory["base_config"]})
4459-
reqheaders = {
4460-
'Content-Type': 'application/json',
4461-
'Content-Length': str(len(reqbody)),
4462-
}
4463-
if args.adminpassword:
4464-
reqheaders["Authorization"] = f"Bearer {args.adminpassword}"
4465-
conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600)
4466-
conn.request("POST", "/api/admin/reload_config", body=reqbody, headers=reqheaders)
4467-
resp = conn.getresponse()
4468-
time.sleep(3)
4469-
global_memory["last_active_timestamp"] = datetime.now()
4470-
if not self.wait_for_upstream_ready(upstream_port,120,0.5):
4471-
self.send_error(504, "KoboldCpp model swap reload timed out")
4472-
return
4473-
time.sleep(0.1)
4456+
reqbody = json.dumps({"filename":global_memory["current_model"], "baseconfig": global_memory["base_config"]})
4457+
reqheaders = {
4458+
'Content-Type': 'application/json',
4459+
'Content-Length': str(len(reqbody)),
4460+
}
4461+
if args.adminpassword:
4462+
reqheaders["Authorization"] = f"Bearer {args.adminpassword}"
4463+
conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600)
4464+
conn.request("POST", "/api/admin/reload_config", body=reqbody, headers=reqheaders)
4465+
resp = conn.getresponse()
4466+
time.sleep(3)
4467+
global_memory["last_active_timestamp"] = datetime.now()
4468+
if not self.wait_for_upstream_ready(upstream_port,120,0.5):
4469+
self.send_error(504, "KoboldCpp model swap reload timed out")
4470+
return
4471+
time.sleep(0.1)
44744472

44754473
try: # connect upstream
44764474
conn = http.client.HTTPConnection('localhost', upstream_port, timeout=600)

0 commit comments

Comments
 (0)