hadadrjt commited on
Commit
a4cdda6
·
1 Parent(s): c7c62f6

fixup! ai: Better handling of load balancing.

Browse files
Files changed (1) hide show
  1. jarvis.py +3 -0
jarvis.py CHANGED
@@ -207,6 +207,9 @@ async def fetch_response_stream_async(host, key, model, msgs, cfg, sid, stop_eve
207
  try:
208
  async with httpx.AsyncClient(timeout=t) as client:
209
  async with client.stream("POST", host, json={**{"model": model, "messages": msgs, "session_id": sid, "stream": True}, **cfg}, headers={"Authorization": f"Bearer {key}"}) as response:
 
 
 
210
  async for line in response.aiter_lines():
211
  if stop_event.is_set():
212
  return
 
207
  try:
208
  async with httpx.AsyncClient(timeout=t) as client:
209
  async with client.stream("POST", host, json={**{"model": model, "messages": msgs, "session_id": sid, "stream": True}, **cfg}, headers={"Authorization": f"Bearer {key}"}) as response:
210
+ if response.status_code in LINUX_SERVER_ERRORS:
211
+ marked_item(key, LINUX_SERVER_PROVIDER_KEYS_MARKED, LINUX_SERVER_PROVIDER_KEYS_ATTEMPTS)
212
+ return
213
  async for line in response.aiter_lines():
214
  if stop_event.is_set():
215
  return