From 2ee599b848302ad71325985c842bc30c319bfca8 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Tue, 28 May 2024 22:08:06 -0700
Subject: [PATCH] docs(batching.md): add batch completion to docs

---
 docs/my-website/docs/completion/batching.md | 71 ++++++++++++++++++++-
 1 file changed, 70 insertions(+), 1 deletion(-)

diff --git a/docs/my-website/docs/completion/batching.md b/docs/my-website/docs/completion/batching.md
index 09f59f743d0..313356b7e25 100644
--- a/docs/my-website/docs/completion/batching.md
+++ b/docs/my-website/docs/completion/batching.md
@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Batching Completion()
 LiteLLM allows you to:
 * Send many completion calls to 1 model
@@ -51,6 +54,9 @@ This makes parallel calls to the specified `models` and returns the first respon
 
 Use this to reduce latency
 
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
 ### Example Code
 ```python
 import litellm
@@ -68,8 +74,70 @@ response = batch_completion_models(
 print(result)
 ```
 
+
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+[how to setup proxy config](../proxy/configs.md)
+
+Just pass a comma-separated string of model names and the flag `fastest_response=True`.
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash
+
+curl -X POST 'http://localhost:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \ 
+-D '{
+    "model": "gpt-3.5-turbo, bedrock-anthropic-claude-3", # 👈 Comma-separated models
+    "messages": [
+      {
+        "role": "user",
+        "content": "What'\''s the weather like in Boston today?"
+      }
+    ],
+    "stream": true,
+    "fastest_response": true # 👈 FLAG
+}
+
+'
+```
+
+</TabItem>
+<TabItem value="openai" label="OpenAI SDK">
+
+```python
+import openai
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+# request sent to model set on litellm proxy, `litellm --model`
+response = client.chat.completions.create(
+    model="gpt-3.5-turbo, bedrock-anthropic-claude-3", # 👈 Comma-separated models
+    messages = [
+        {
+            "role": "user",
+            "content": "this is a test request, write a short poem"
+        }
+    ],
+    extra_body={"fastest_response": true} # 👈 FLAG
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+</TabItem>
+</Tabs>
+
 ### Output
-Returns the first response
+Returns the first response in OpenAI format. Cancels other LLM API calls. 
 ```json
 {
   "object": "chat.completion",
@@ -95,6 +163,7 @@ Returns the first response
 }
 ```
 
+
 ## Send 1 completion call to many models: Return All Responses
 This makes parallel calls to the specified models and returns all responses