From f1dc9a4ab85116f486eb3767a99bc89f2755e77a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 22 Jan 2026 19:00:56 -0800 Subject: [PATCH] docs: update tutorial --- .../docs/guides/anthropic_background_mode.md | 145 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/proxy/_new_secret_config.yaml | 7 + litellm/utils.py | 8 +- 4 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/guides/anthropic_background_mode.md diff --git a/docs/my-website/docs/guides/anthropic_background_mode.md b/docs/my-website/docs/guides/anthropic_background_mode.md new file mode 100644 index 00000000000..21f6f282e0e --- /dev/null +++ b/docs/my-website/docs/guides/anthropic_background_mode.md @@ -0,0 +1,145 @@ +# Stream OpenAI/Anthropic/Gemini/etc. in Ruby + +Imaging the case where services are running frameworks that does not support multithreading (for example, your company runs ruby service on single thread framework), you want to stream responses, but don't want to block I/O for others since it is a single thread. Native SSE streaming won't work in this case. LiteLLM supports background mode where user can use litellm to poll partial responses periodically to simulate streaming. + + +## How it works + +Think of this as a producer–consumer pattern: + +**Producer**: LiteLLM runs the LLM request in the background and incrementally writes chunks to Redis. + +**Consumer**: Your app periodically polls LiteLLM to fetch the latest partial response. + +## Usage + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-sonnet-4-5-20250929 + litellm_params: + model: anthropic/claude-sonnet-4-5-20250929 + +litellm_settings: + cache: true + cache_params: + type: redis + ttl: 3600 + host: "127.0.0.1" + port: "6379" + responses: + background_mode: + polling_via_cache: "all" + ttl: 3600 +``` + +2. Start LiteLLM Proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Make request + +```bash +curl http://0.0.0.0:4000/v1/responses \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "claude-sonnet-4-5-20250929", + "input": "Tell me a three sentence bedtime story about a unicorn.", + "background": true + }' +``` + +Expected Response: + +```bash +{ + "id": "litellm_poll_adff0089-f9d3-4135-b49e-f582597deedd", + "object": "response", + "status": "queued", + "output": [], + "usage": null, + "metadata": {}, + "created_at": 1764913614 +} +``` + +Get response curl cmd: + +```bash +curl http://0.0.0.0:4000/v1/responses/litellm_poll_adff0089-f9d3-4135-b49e-f582597deedd \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" +``` + +Response: + +Keep polling until the response is completed. + +```bash +{ + "id": "litellm_poll_adff0089-f9d3-4135-b49e-f582597deedd", + "created_at": 1764913614, + "error": null, + "incomplete_details": null, + "instructions": null, + "metadata": {}, + "model": "claude-sonnet-4-5-20250929", + "object": "response", + "output": [ + { + "id": "rs_0b467fde531ff8fb00693271cf4ec88196aee3b64030ecc23d", + "summary": [], + "type": "reasoning" + }, + { + "id": "msg_0b467fde531ff8fb00693271d2885c8196b2ccf625b4560125", + "content": [ + { + "annotations": [], + "text": "Under a moonlit sky, a gentle unicorn named Lumen wandered through a whispering forest, leaving silver hoofprints that glowed softly in the moss. She paused beside a sleepy brook and lowered her horn, and the ripples turned into twinkling stars that drifted up to join the night. With the forest tucked beneath a quilt of starlight, Lumen closed her eyes and dreamed of new paths to light when morning came.", + "type": "output_text", + "logprobs": [] + } + ], + "role": "assistant", + "status": "completed", + "type": "message" + } + ], + "parallel_tool_calls": true, + "temperature": 1, + "tool_choice": "auto", + "tools": [], + "top_p": 1, + "max_output_tokens": null, + "previous_response_id": null, + "reasoning": { + "effort": "medium" + }, + "status": "completed", + "text": { + "format": { + "type": "text" + }, + "verbosity": "medium" + }, + "truncation": "disabled", + "usage": { + "input_tokens": 17, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 287, + "output_tokens_details": { + "reasoning_tokens": 192 + }, + "total_tokens": 304 + }, + "user": null, + "store": true +} +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index ad5019d880a..859b5dc6735 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -857,6 +857,7 @@ const sidebars = { "completion/batching", "guides/finetuned_models", "guides/security_settings", + "guides/anthropic_background_mode", "proxy/veo_video_generation", "reasoning_content", "extras/creating_adapters", diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 13eeae14485..60dbd4ee9a6 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -14,3 +14,10 @@ model_list: litellm_params: model: openai/gpt-4.1-mini + +litellm_settings: + cache: true + responses: + background_mode: + polling_via_cache: "all" + ttl: 3600 \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index c56c49cb995..9bb879411a1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1398,6 +1398,10 @@ def client(original_function): # noqa: PLR0915 @wraps(original_function) def wrapper(*args, **kwargs): # noqa: PLR0915 + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObject, + ) + # DO NOT MOVE THIS. It always needs to run first # Check if this is an async function. If so only execute the async function call_type = original_function.__name__ @@ -1745,7 +1749,9 @@ def client(original_function): # noqa: PLR0915 @wraps(original_function) async def wrapper_async(*args, **kwargs): # noqa: PLR0915 - from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObject, + ) print_args_passed_to_litellm(original_function, args, kwargs) start_time = datetime.datetime.now()