From 904a7b85ac57babe025807b24af63f92d46fd5da Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 22 Jan 2026 19:18:36 -0800 Subject: [PATCH] docs(anthropic_background_mode.md): guide on when to use responses api background mode via litellm --- cookbook/ai_coding_tool_guides/index.json | 15 +++++++++++++++ .../docs/guides/anthropic_background_mode.md | 6 +++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cookbook/ai_coding_tool_guides/index.json b/cookbook/ai_coding_tool_guides/index.json index 3e71670d623..3c8934ef00a 100644 --- a/cookbook/ai_coding_tool_guides/index.json +++ b/cookbook/ai_coding_tool_guides/index.json @@ -131,4 +131,19 @@ "Claude Code", "LiteLLM" ] +}, +{ + "title": "Stream OpenAI/Anthropic/Gemini/etc. in Ruby", + "description": "This is a guide to streaming OpenAI/Anthropic/Gemini/etc. in Ruby or other single-threaded frameworks.", + "url": "https://docs.litellm.ai/docs/guides/anthropic_background_mode", + "date": "2026-01-22", + "version": "1.0.0", + "tags": [ + "OpenAI", + "Anthropic", + "Gemini", + "Ruby", + "Responses API", + "Streaming" + ] }] \ No newline at end of file diff --git a/docs/my-website/docs/guides/anthropic_background_mode.md b/docs/my-website/docs/guides/anthropic_background_mode.md index 21f6f282e0e..39aced3a02b 100644 --- a/docs/my-website/docs/guides/anthropic_background_mode.md +++ b/docs/my-website/docs/guides/anthropic_background_mode.md @@ -3,6 +3,10 @@ Imaging the case where services are running frameworks that does not support multithreading (for example, your company runs ruby service on single thread framework), you want to stream responses, but don't want to block I/O for others since it is a single thread. Native SSE streaming won't work in this case. LiteLLM supports background mode where user can use litellm to poll partial responses periodically to simulate streaming. +Requirements: +- Redis cache must be enabled +- /responses API endpoint must be used + ## How it works Think of this as a producer–consumer pattern: @@ -77,7 +81,7 @@ curl http://0.0.0.0:4000/v1/responses/litellm_poll_adff0089-f9d3-4135-b49e-f5825 Response: -Keep polling until the response is completed. +**Keep polling until the response is completed.** ```bash {