More resilient eval (#32257)
Bubbles up rate limit information so that we can retry after a certain duration if needed higher up in the stack. Also caps the number of concurrent evals running at once to also help. Release Notes: - N/A
This commit is contained in:
@@ -386,8 +386,10 @@ impl CodegenAlternative {
|
||||
async { Ok(LanguageModelTextStream::default()) }.boxed_local()
|
||||
} else {
|
||||
let request = self.build_request(&model, user_prompt, cx)?;
|
||||
cx.spawn(async move |_, cx| model.stream_completion_text(request.await, &cx).await)
|
||||
.boxed_local()
|
||||
cx.spawn(async move |_, cx| {
|
||||
Ok(model.stream_completion_text(request.await, &cx).await?)
|
||||
})
|
||||
.boxed_local()
|
||||
};
|
||||
self.handle_stream(telemetry_id, provider_id.to_string(), api_key, stream, cx);
|
||||
Ok(())
|
||||
|
||||
@@ -1563,6 +1563,9 @@ impl Thread {
|
||||
Err(LanguageModelCompletionError::Other(error)) => {
|
||||
return Err(error);
|
||||
}
|
||||
Err(err @ LanguageModelCompletionError::RateLimit(..)) => {
|
||||
return Err(err.into());
|
||||
}
|
||||
};
|
||||
|
||||
match event {
|
||||
|
||||
Reference in New Issue
Block a user