From 595f1d4802107566dac4eb0df53c6818c70a7305 Mon Sep 17 00:00:00 2001 From: Anurag Kumar Date: Mon, 23 Mar 2026 11:05:49 +0000 Subject: [PATCH] Clarify --- docs/my-website/docs/proxy/config_settings.md | 2 +- docs/my-website/docs/proxy/request_headers.md | 3 +-- docs/my-website/docs/proxy/timeout.md | 16 ++++++++-------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 4b902738924..d267caedc27 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -341,7 +341,7 @@ router_settings: | max_fallbacks | Optional[int] | The maximum number of fallbacks to try before exiting the call. Defaults to 5. | | default_litellm_params | Optional[dict] | The default litellm parameters to add to all requests (e.g. `temperature`, `max_tokens`). | | timeout | Optional[float] | The default timeout for a request. Default is 10 minutes. | -| stream_timeout | Optional[float] | The default timeout between streamed reads / chunks. If not set, the normal `timeout` value is used. | +| stream_timeout | Optional[float] | The default timeout used for `stream=True` calls. In practice this is the timeout between streamed reads / chunks. If not set, the normal `timeout` value is used. | | debug_level | Literal["DEBUG", "INFO"] | The debug level for the logging library in the router. Defaults to "INFO". | | client_ttl | int | Time-to-live for cached clients in seconds. Defaults to 3600. | | cache_kwargs | dict | Additional keyword arguments for the cache initialization. Use this for non-string Redis parameters that may fail when set via `REDIS_*` environment variables. | diff --git a/docs/my-website/docs/proxy/request_headers.md b/docs/my-website/docs/proxy/request_headers.md index c4fac8fec30..f60aea1ec29 100644 --- a/docs/my-website/docs/proxy/request_headers.md +++ b/docs/my-website/docs/proxy/request_headers.md @@ -10,7 +10,7 @@ By default, LiteLLM does not forward client headers to LLM provider APIs. Howeve `x-litellm-timeout` Optional[float]: The timeout for the request in seconds. -`x-litellm-stream-timeout` Optional[float]: The timeout in seconds between streamed reads / chunks (only applies for streaming requests). If a stream pauses longer than this, LiteLLM times out. If not set, LiteLLM uses the normal request timeout. [Demo Video](https://www.loom.com/share/8da67e4845ce431a98c901d4e45db0e5) +`x-litellm-stream-timeout` Optional[float]: Timeout used for `stream=True` calls. In practice this is the timeout in seconds between streamed reads / chunks. If a stream pauses longer than this, LiteLLM times out. If not set, LiteLLM uses the normal request timeout. [Demo Video](https://www.loom.com/share/8da67e4845ce431a98c901d4e45db0e5) `x-litellm-enable-message-redaction`: Optional[bool]: Don't log the message content to logging integrations. Just track spend. [Learn More](./logging#redact-messages-response-content) @@ -39,4 +39,3 @@ By default, LiteLLM does not forward client headers to LLM provider APIs. Howeve Custom headers starting with `x-` can be forwarded to LLM provider APIs when the model is configured in `forward_client_headers_to_llm_api`. [Learn more about header forwarding configuration](./forward_client_headers.md). - diff --git a/docs/my-website/docs/proxy/timeout.md b/docs/my-website/docs/proxy/timeout.md index b67a9b0e744..3bfd99b55b1 100644 --- a/docs/my-website/docs/proxy/timeout.md +++ b/docs/my-website/docs/proxy/timeout.md @@ -42,26 +42,26 @@ $ litellm --config /path/to/config.yaml For each model, you can set `timeout` and `stream_timeout` under `litellm_params`: -- **`timeout`** → maximum time for the *complete request*. - Use this to cap the total end-to-end call duration. +- **`timeout`** → maximum time for the *complete request*. Use this to cap the total end-to-end call duration. -- **`stream_timeout`** → maximum time to wait for the next streamed read / chunk before timing out. - Use this when a provider starts streaming and then stalls, or when a remote setup has long gaps between streamed chunks. +- **`stream_timeout`** → timeout used for `stream=True` calls. With LiteLLM's default httpx client, this is how long LiteLLM waits for the next streamed read / chunk before timing out. If `stream_timeout` is not set, LiteLLM falls back to the normal `timeout` value. ### Practical rule of thumb - Use **`timeout`** to control how long the whole request is allowed to run. -- Use **`stream_timeout`** to control how long LiteLLM waits between streamed chunks. +- Use **`stream_timeout`** to control how quickly a stalled stream should fail. - If you are debugging socket read timeouts or mid-stream disconnects in a remote setup, start by setting **both** values to the same number. - Once the stream is stable, lower `stream_timeout` only if you want faster failover for stalled streams. +- For quick local smoke tests, sub-second values can still be useful. ### Recommended starting values - **Remote / self-hosted streaming path** (`Open WebUI -> LiteLLM -> provider`, `OpenHands -> LiteLLM -> provider`, etc.): start with `timeout: 180` and `stream_timeout: 180`. - **Normal interactive streaming**: start with `timeout: 180-300` and `stream_timeout: 30-60`. - **Batch / non-streaming requests**: set `timeout`, and you can usually skip `stream_timeout`. + @@ -77,7 +77,7 @@ model_list = [{ "api_version": os.getenv("AZURE_API_VERSION"), "api_base": os.getenv("AZURE_API_BASE"), "timeout": 300, # total request timeout - "stream_timeout": 60 # max gap between streamed chunks + "stream_timeout": 60 # stream=True timeout while waiting for next chunk } }] @@ -105,7 +105,7 @@ model_list: api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ api_key: timeout: 180 # total request timeout (seconds) - stream_timeout: 60 # max gap between streamed chunks (seconds) + stream_timeout: 60 # stream=True timeout while waiting for next chunk (seconds) max_retries: 5 - model_name: gpt-3.5-turbo litellm_params: @@ -113,7 +113,7 @@ model_list: api_base: https://my-endpoint-canada-berri992.openai.azure.com/ api_key: timeout: 180 # total request timeout (seconds) - stream_timeout: 60 # max gap between streamed chunks (seconds) + stream_timeout: 60 # stream=True timeout while waiting for next chunk (seconds) max_retries: 5 ```