mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(router): answer a context window error when the request num_retries cannot be read
A request num_retries the retry loop cannot compare raised the loop's TypeError as a 500 once a context window error became retryable. Hand the context window error on instead, as the request got before.
This commit is contained in:
parent
abbd9287cd
commit
bb0da5feb3
2 changed files with 28 additions and 2 deletions
|
|
@ -1098,17 +1098,20 @@ impl Core {
|
|||
});
|
||||
}
|
||||
if uncounted == Some(UncountedRetries::Raises) {
|
||||
if let Some(refusal) = refusal {
|
||||
return Err(refusal);
|
||||
}
|
||||
self.host.observe(&RouterEvent::RetryCountRaised {
|
||||
level: at.level,
|
||||
model: target.group.clone(),
|
||||
cause: RetryCause::of(&failed.error),
|
||||
});
|
||||
return Err(refusal.unwrap_or(Failed {
|
||||
return Err(Failed {
|
||||
error: Box::new(Error::Attempt(AttemptError::host_raised(&["Exception"]))),
|
||||
deployment_id: None,
|
||||
routed: None,
|
||||
served_as: None,
|
||||
}));
|
||||
});
|
||||
}
|
||||
if num_retries == 0 {
|
||||
return Err(refusal.unwrap_or(failed));
|
||||
|
|
|
|||
|
|
@ -1106,6 +1106,29 @@ async fn a_retry_that_fails_after_a_context_window_error_raises_that_error(confi
|
|||
assert_eq!(host.selected().len(), 3);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_num_retries_that_raises_leaves_a_context_window_error_standing(config: RouterConfig) {
|
||||
let host = Arc::new(ScriptedHost::failing("primary", vec![context_window()]));
|
||||
let router = router(config, json!({"num_retries": 2}), host.clone());
|
||||
let request = RouterRequest {
|
||||
uncounted_retries: Some(UncountedRetries::Raises),
|
||||
..request()
|
||||
};
|
||||
let refused = router.call(Op::from("acompletion"), request).await;
|
||||
assert!(
|
||||
matches!(refused, Err(Error::Attempt(failed)) if failed.kind == FailureKind::ContextWindowExceeded)
|
||||
);
|
||||
assert!(
|
||||
!host
|
||||
.events
|
||||
.lock()
|
||||
.iter()
|
||||
.any(|event| matches!(event, RouterEvent::RetryCountRaised { .. }))
|
||||
);
|
||||
assert_eq!(host.selected().len(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_failed_agentic_followup_is_not_retried_but_still_falls_back(config: RouterConfig) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue