feat(ui): add OCR support to Playground

ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatUI.tsx

@@ -43,6 +43,7 @@ import { Agent, fetchAvailableAgents } from "../../llm_calls/fetch_agents";
 import { fetchAvailableModels, ModelGroup } from "@/components/llm_calls/fetch_models";
 import { makeOpenAIImageEditsRequest } from "../../llm_calls/image_edits";
 import { makeOpenAIImageGenerationRequest } from "../../llm_calls/image_generation";
+import { makeOpenAIOcrRequest } from "../../llm_calls/OcrApi";
 import { makeOpenAIResponsesRequest } from "@/components/llm_calls/responses_api";
 import { makeInteractionsRequest } from "../../llm_calls/interactions_api";
 import AdditionalModelSettings from "./AdditionalModelSettings";
@@ -255,6 +256,8 @@ const ChatUI: React.FC<ChatUIProps> = ({
   const [chatUploadedImage, setChatUploadedImage] = useState<File | null>(null);
   const [chatImagePreviewUrl, setChatImagePreviewUrl] = useState<string | null>(null);
   const [uploadedAudio, setUploadedAudio] = useState<File | null>(null);
+  const [uploadedOcrFile, setUploadedOcrFile] = useState<File | null>(null);
+  const [ocrFilePreviewUrl, setOcrFilePreviewUrl] = useState<string | null>(null);
   const [isGetCodeModalVisible, setIsGetCodeModalVisible] = useState(false);
   const [generatedCode, setGeneratedCode] = useState("");
   const [selectedSdk, setSelectedSdk] = useState<"openai" | "azure">("openai");
@@ -700,8 +703,35 @@ const ChatUI: React.FC<ChatUIProps> = ({
     setUploadedAudio(null);
   };
 
+  const handleOcrFileUpload = (file: File): false => {
+    setUploadedOcrFile(file);
+    setOcrFilePreviewUrl(file.type.startsWith("image/") ? URL.createObjectURL(file) : null);
+    return false;
+  };
+
+  const handleOcrFileInputChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    const file = event.target.files?.[0];
+    if (file) {
+      handleOcrFileUpload(file);
+    }
+    event.target.value = "";
+  };
+
+  const handleRemoveOcrFile = () => {
+    if (ocrFilePreviewUrl) {
+      URL.revokeObjectURL(ocrFilePreviewUrl);
+    }
+    setUploadedOcrFile(null);
+    setOcrFilePreviewUrl(null);
+  };
+
   const handleSendMessage = async () => {
-    if (inputMessage.trim() === "" && endpointType !== EndpointType.TRANSCRIPTION && endpointType !== EndpointType.MCP)
+    if (
+      inputMessage.trim() === "" &&
+      endpointType !== EndpointType.TRANSCRIPTION &&
+      endpointType !== EndpointType.MCP &&
+      endpointType !== EndpointType.OCR
+    )
       return;
 
     // For image edits, require both image and prompt
@@ -716,6 +746,11 @@ const ChatUI: React.FC<ChatUIProps> = ({
       return;
     }
 
+    if (endpointType === EndpointType.OCR && !uploadedOcrFile) {
+      toast.fromError("Please upload a document or image for OCR");
+      return;
+    }
+
     // For A2A agents, require agent selection
     if (endpointType === EndpointType.A2A_AGENTS && !selectedAgent) {
       toast.fromError("Please select an agent to send a message");
@@ -771,6 +806,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
       EndpointType.ANTHROPIC_MESSAGES,
       EndpointType.EMBEDDINGS,
       EndpointType.TRANSCRIPTION,
+      EndpointType.OCR,
       EndpointType.INTERACTIONS,
     ];
 
@@ -783,7 +819,11 @@ const ChatUI: React.FC<ChatUIProps> = ({
       return;
     }
 
-    const effectiveApiKey = simplified ? accessToken : apiKeySource === "session" ? accessToken : apiKey;
+    const effectiveApiKey = (() => {
+      if (simplified) return accessToken;
+      if (apiKeySource === "session") return accessToken;
+      return apiKey;
+    })();
 
     if (!effectiveApiKey) {
       toast.fromError("Please provide a Virtual Key or select Current UI Session");
@@ -846,6 +886,8 @@ const ChatUI: React.FC<ChatUIProps> = ({
         ? `🎵 Audio file: ${uploadedAudio.name}\nPrompt: ${inputMessage}`
         : `🎵 Audio file: ${uploadedAudio.name}`;
       displayMessage = createDisplayMessage(audioMessage, false);
+    } else if (endpointType === EndpointType.OCR && uploadedOcrFile) {
+      displayMessage = createDisplayMessage(`OCR file: ${uploadedOcrFile.name}`, false);
     } else if (endpointType === EndpointType.MCP && selectedMCPDirectTool) {
       // For MCP direct mode, show tool name and arguments from form
       const mcpMessage = `🔧 MCP Tool: ${selectedMCPDirectTool}\nArguments: ${JSON.stringify(mcpToolArguments, null, 2)}`;
@@ -1043,6 +1085,18 @@ const ChatUI: React.FC<ChatUIProps> = ({
               customProxyBaseUrl || undefined,
             );
           }
+        } else if (endpointType === EndpointType.OCR) {
+          if (uploadedOcrFile) {
+            await makeOpenAIOcrRequest({
+              file: uploadedOcrFile,
+              updateUI: (text, model) => updateTextUI("assistant", text, model),
+              selectedModel,
+              accessToken: effectiveApiKey,
+              tags: selectedTags,
+              signal,
+              customBaseUrl: customProxyBaseUrl || undefined,
+            });
+          }
         } else if (endpointType === EndpointType.INTERACTIONS) {
           await makeInteractionsRequest(
             inputMessage,
@@ -1128,6 +1182,9 @@ const ChatUI: React.FC<ChatUIProps> = ({
       if (endpointType === EndpointType.TRANSCRIPTION && uploadedAudio) {
         handleRemoveAudio();
       }
+      if (endpointType === EndpointType.OCR && uploadedOcrFile) {
+        handleRemoveOcrFile();
+      }
     }
 
     setInputMessage("");
@@ -1139,6 +1196,7 @@ const ChatUI: React.FC<ChatUIProps> = ({
     handleRemoveResponsesImage();
     handleRemoveChatImage();
     handleRemoveAudio();
+    handleRemoveOcrFile();
     toast.success("Chat history cleared.");
   };
 
@@ -1194,7 +1252,9 @@ const ChatUI: React.FC<ChatUIProps> = ({
             ? "Enter text to convert to speech..."
             : endpointType === EndpointType.TRANSCRIPTION
               ? "Optional: Add context or prompt for transcription..."
-              : "Describe the image you want to generate...";
+              : endpointType === EndpointType.OCR
+                ? "Upload a document or image to run OCR"
+                : "Describe the image you want to generate...";
 
   const sendDisabled =
     isLoading ||
@@ -1202,7 +1262,9 @@ const ChatUI: React.FC<ChatUIProps> = ({
       ? !(selectedMCPServers.length === 1 && selectedMCPServers[0] !== "__all__" && selectedMCPDirectTool)
       : endpointType === EndpointType.TRANSCRIPTION
         ? !uploadedAudio
-        : !inputMessage.trim());
+        : endpointType === EndpointType.OCR
+          ? !uploadedOcrFile
+          : !inputMessage.trim());
 
   return (
     <div className={`min-h-0 min-w-0 bg-white ${simplified ? "flex h-full w-full flex-col" : "h-full w-full p-3"}`}>
@@ -1927,6 +1989,31 @@ const ChatUI: React.FC<ChatUIProps> = ({
                     </div>
                   )}
 
+                  {endpointType === EndpointType.OCR && !uploadedOcrFile && (
+                    <div className="mb-4">
+                      <label
+                        className="flex cursor-pointer flex-col items-center justify-center rounded-lg border-2 border-dashed border-gray-300 bg-gray-50 px-4 py-8 text-center hover:border-gray-400"
+                        onDragOver={(event) => event.preventDefault()}
+                        onDrop={(event) => {
+                          event.preventDefault();
+                          const file = event.dataTransfer.files[0];
+                          if (file) {
+                            handleOcrFileUpload(file);
+                          }
+                        }}
+                      >
+                        <ImageIcon className="mb-2 size-6 text-gray-500" aria-hidden="true" />
+                        <p className="text-sm">Click or drag a document or image to upload</p>
+                        <p className="text-xs text-gray-500">Support for PDF and image files.</p>
+                        <input
+                          type="file"
+                          accept="image/*,application/pdf"
+                          className="sr-only"
+                          onChange={handleOcrFileInputChange}
+                        />
+                      </label>
+                    </div>
+                  )}
                   {endpointType === EndpointType.RESPONSES && responsesUploadedImage && (
                     <FilePreviewCard
                       file={responsesUploadedImage}
@@ -1943,6 +2030,13 @@ const ChatUI: React.FC<ChatUIProps> = ({
                     />
                   )}
 
+                  {endpointType === EndpointType.OCR && uploadedOcrFile && (
+                    <FilePreviewCard
+                      file={uploadedOcrFile}
+                      previewUrl={ocrFilePreviewUrl}
+                      onRemove={handleRemoveOcrFile}
+                    />
+                  )}
                   {endpointType === EndpointType.RESPONSES && codeInterpreter.enabled && (
                     <div className="mb-2 space-y-2">
                       <div className="flex items-center justify-between rounded-lg border border-blue-200 bg-linear-to-r from-blue-50 to-purple-50 px-3 py-2">
@@ -1994,10 +2088,15 @@ const ChatUI: React.FC<ChatUIProps> = ({
                     onSubmit={handleSendMessage}
                     onCancel={handleCancelRequest}
                     placeholder={inputPlaceholder}
-                    disabled={isLoading}
+                    disabled={isLoading || endpointType === EndpointType.OCR}
                     isLoading={isLoading}
                     submitDisabled={sendDisabled}
-                    showSuggestions={chatHistory.length === 0 && !isLoading && endpointType !== EndpointType.MCP}
+                    showSuggestions={
+                      chatHistory.length === 0 &&
+                      !isLoading &&
+                      endpointType !== EndpointType.MCP &&
+                      endpointType !== EndpointType.OCR
+                    }
                     suggestions={
                       endpointType === EndpointType.A2A_AGENTS
                         ? ["What can you help me with?", "Tell me about yourself", "What tasks can you perform?"]

ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/EndpointSelector.test.tsx

@@ -26,4 +26,17 @@ describe("EndpointSelector", () => {
     expect(await screen.findByText("/v1/audio/speech")).toBeInTheDocument();
     expect(await screen.findByText("/v1/audio/transcriptions")).toBeInTheDocument();
   });
+
+  it("should filter and show the OCR endpoint when user inputs 'ocr'", async () => {
+    const user = userEvent.setup();
+    render(<EndpointSelector endpointType={ENDPOINT_OPTIONS[0].value} onEndpointChange={() => {}} />);
+
+    const combobox = screen.getByRole("combobox");
+    await user.click(combobox);
+
+    const input = await screen.findByRole("combobox");
+    await user.type(input, "ocr");
+
+    expect(await screen.findByText("/v1/ocr")).toBeInTheDocument();
+  });
 });

ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/chatConstants.ts

@@ -42,6 +42,7 @@ export const ENDPOINT_OPTIONS = [
   { value: EndpointType.EMBEDDINGS, label: "/v1/embeddings" },
   { value: EndpointType.SPEECH, label: "/v1/audio/speech" },
   { value: EndpointType.TRANSCRIPTION, label: "/v1/audio/transcriptions" },
+  { value: EndpointType.OCR, label: "/v1/ocr" },
   { value: EndpointType.A2A_AGENTS, label: "/v1/a2a/message/send" },
   { value: EndpointType.MCP, label: "/mcp-rest/tools/call" },
   { value: EndpointType.REALTIME, label: "/v1/realtime" },

ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/OcrApi.test.tsx

@@ -0,0 +1,62 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { makeOpenAIOcrRequest } from "./OcrApi";
+
+vi.mock("@/components/networking", () => ({
+  getProxyBaseUrl: vi.fn(() => "https://example.com"),
+  getGlobalLitellmHeaderName: vi.fn(() => "Authorization"),
+}));
+
+describe("ocr_api", () => {
+  const mockUpdateUI = vi.fn();
+  const mockFetch = vi.fn();
+
+  beforeEach(() => {
+    const responseBody = JSON.stringify({
+      pages: [
+        {
+          index: 0,
+          markdown: "# Extracted text",
+        },
+      ],
+      model: "mistral-ocr-latest",
+    });
+    mockFetch.mockResolvedValue({
+      ok: true,
+      text: async () => responseBody,
+    } as Response);
+
+    global.fetch = mockFetch;
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("posts OCR uploads to /v1/ocr with model and file multipart fields", async () => {
+    const file = new File(["document data"], "invoice.pdf", { type: "application/pdf" });
+
+    await makeOpenAIOcrRequest({
+      file,
+      updateUI: mockUpdateUI,
+      selectedModel: "mistral-ocr-latest",
+      accessToken: "sk-1234567890",
+      tags: ["tag1", "tag2"],
+    });
+
+    expect(mockFetch).toHaveBeenCalledTimes(1);
+    expect(mockFetch).toHaveBeenCalledWith("https://example.com/v1/ocr", {
+      method: "POST",
+      headers: {
+        Authorization: "Bearer sk-1234567890",
+        "x-litellm-tags": "tag1,tag2",
+      },
+      body: expect.any(FormData),
+      signal: undefined,
+    });
+
+    const formData = mockFetch.mock.calls[0][1].body as FormData;
+    expect(formData.get("model")).toBe("mistral-ocr-latest");
+    expect(formData.get("file")).toBe(file);
+    expect(mockUpdateUI).toHaveBeenCalledWith("# Extracted text", "mistral-ocr-latest");
+  });
+});

ui/litellm-dashboard/src/app/(dashboard)/playground/llm_calls/OcrApi.tsx

@@ -0,0 +1,66 @@
+import NotificationManager from "@/components/molecules/notifications_manager";
+import { getGlobalLitellmHeaderName, getProxyBaseUrl } from "@/components/networking";
+import { createApiClient } from "@/lib/http/client";
+
+interface OcrRequestParams {
+  file: File;
+  updateUI: (text: string, model: string) => void;
+  selectedModel: string;
+  accessToken: string;
+  tags?: string[];
+  signal?: AbortSignal;
+  customBaseUrl?: string;
+}
+
+const formatOcrResponse = (response: unknown): string => {
+  if (typeof response !== "object" || response === null || !("pages" in response)) {
+    return JSON.stringify(response, null, 2);
+  }
+
+  const pages = (response as { pages?: unknown }).pages;
+  if (!Array.isArray(pages)) {
+    return JSON.stringify(response, null, 2);
+  }
+
+  const markdown = pages
+    .map((page) =>
+      typeof page === "object" && page !== null && "markdown" in page
+        ? (page as { markdown?: unknown }).markdown
+        : undefined,
+    )
+    .filter((pageMarkdown): pageMarkdown is string => typeof pageMarkdown === "string" && pageMarkdown.length > 0)
+    .join("\n\n");
+
+  return markdown || JSON.stringify(response, null, 2);
+};
+
+export async function makeOpenAIOcrRequest({
+  file,
+  updateUI,
+  selectedModel,
+  accessToken,
+  tags,
+  signal,
+  customBaseUrl,
+}: OcrRequestParams) {
+  const client = createApiClient({
+    getBaseUrl: () => customBaseUrl || getProxyBaseUrl(),
+    getAuthHeaderName: getGlobalLitellmHeaderName,
+    onError: (message) => NotificationManager.fromBackend(`OCR failed: ${message}`),
+  });
+  const formData = new FormData();
+  formData.append("model", selectedModel);
+  formData.append("file", file);
+
+  const responseJson = await client.post<unknown>("/v1/ocr", {
+    accessToken,
+    rawBody: formData,
+    headers: {
+      ...(tags && tags.length > 0 ? { "x-litellm-tags": tags.join(",") } : {}),
+    },
+    signal,
+  });
+
+  updateUI(formatOcrResponse(responseJson), selectedModel);
+  NotificationManager.success("OCR completed successfully");
+}

ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.test.ts

@@ -0,0 +1,16 @@
+import { describe, expect, it } from "vitest";
+import { EndpointType, getEndpointType, litellmModeMapping, ModelMode } from "./mode_endpoint_mapping";
+
+describe("mode_endpoint_mapping", () => {
+  it("maps OCR model mode to the OCR endpoint", () => {
+    expect(getEndpointType("ocr")).toBe(EndpointType.OCR);
+    expect(litellmModeMapping[ModelMode.OCR]).toBe(EndpointType.OCR);
+  });
+
+  it("preserves existing model mode mappings", () => {
+    expect(getEndpointType("chat")).toBe(EndpointType.CHAT);
+    expect(getEndpointType("responses")).toBe(EndpointType.RESPONSES);
+    expect(getEndpointType("image_generation")).toBe(EndpointType.IMAGE);
+    expect(getEndpointType("audio_transcription")).toBe(EndpointType.TRANSCRIPTION);
+  });
+});

ui/litellm-dashboard/src/components/chat_ui/mode_endpoint_mapping.tsx

@@ -12,6 +12,7 @@ export enum ModelMode {
   ANTHROPIC_MESSAGES = "anthropic_messages",
   EMBEDDING = "embedding",
   REALTIME = "realtime",
+  OCR = "ocr",
 }
 
 // Define an enum for the endpoint types your UI calls
@@ -25,6 +26,7 @@ export enum EndpointType {
   EMBEDDINGS = "embeddings",
   SPEECH = "speech",
   TRANSCRIPTION = "transcription",
+  OCR = "ocr",
   A2A_AGENTS = "a2a_agents",
   MCP = "mcp",
   REALTIME = "realtime",
@@ -43,6 +45,7 @@ export const litellmModeMapping: Record<ModelMode, EndpointType> = {
   [ModelMode.AUDIO_TRANSCRIPTION]: EndpointType.TRANSCRIPTION,
   [ModelMode.EMBEDDING]: EndpointType.EMBEDDINGS,
   [ModelMode.REALTIME]: EndpointType.REALTIME,
+  [ModelMode.OCR]: EndpointType.OCR,
 };
 
 export const getEndpointType = (mode: string): EndpointType => {