diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 4a6cfdf1a33..8171a9dd045 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -589,8 +589,10 @@ response = litellm.completion( "content": [ {"type": "text", "text": "Please summarize the audio."}, { - "type": "image_url", - "image_url": "data:audio/mp3;base64,{}".format(encoded_data), # 👈 SET MIME_TYPE + DATA + "type": "file", + "file": { + "file_data": "data:audio/mp3;base64,{}".format(encoded_data), # 👈 SET MIME_TYPE + DATA + } }, ], } @@ -640,8 +642,11 @@ response = litellm.completion( "content": [ {"type": "text", "text": "Please summarize the file."}, { - "type": "image_url", - "image_url": "https://storage..." # 👈 SET THE IMG URL + "type": "file", + "file": { + "file_id": "https://storage...", # 👈 SET THE IMG URL + "format": "application/pdf" # OPTIONAL + } }, ], } @@ -668,8 +673,11 @@ response = litellm.completion( "content": [ {"type": "text", "text": "Please summarize the file."}, { - "type": "image_url", - "image_url": "gs://..." # 👈 SET THE cloud storage bucket url + "type": "file", + "file": { + "file_id": "gs://storage...", # 👈 SET THE IMG URL + "format": "application/pdf" # OPTIONAL + } }, ], } diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 103cc07ef55..2865a82ce12 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -1720,23 +1720,25 @@ assert isinstance( ``` -## Usage - PDF / Videos / etc. Files +## Usage - PDF / Videos / Audio etc. Files Pass any file supported by Vertex AI, through LiteLLM. -LiteLLM Supports the following image types passed in url +LiteLLM Supports the following file types passed in url. + +Using `file` message type for VertexAI is live from v1.65.1+ ``` -Images with Cloud Storage URIs - gs://cloud-samples-data/generative-ai/image/boats.jpeg -Images with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg +Files with Cloud Storage URIs - gs://cloud-samples-data/generative-ai/image/boats.jpeg +Files with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg Videos with Cloud Storage URIs - https://storage.googleapis.com/github-repo/img/gemini/multimodality_usecases_overview/pixel8.mp4 -Base64 Encoded Local Images +Base64 Encoded Local Files ``` -### **Using `gs://`** +### **Using `gs://` or any URL** ```python from litellm import completion @@ -1748,8 +1750,11 @@ response = completion( "content": [ {"type": "text", "text": "You are a very professional document summarization specialist. Please summarize the given document."}, { - "type": "image_url", - "image_url": "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf", # 👈 PDF + "type": "file", + "file": { + "file_id": "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf", + "format": "application/pdf" # OPTIONAL - specify mime-type + } }, ], } @@ -1783,8 +1788,10 @@ response = completion( "content": [ {"type": "text", "text": "You are a very professional document summarization specialist. Please summarize the given document."}, { - "type": "image_url", - "image_url": f"data:application/pdf;base64,{encoded_file}", # 👈 PDF + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{encoded_file}", # 👈 PDF + } }, ], } @@ -1830,8 +1837,11 @@ curl http://0.0.0.0:4000/v1/chat/completions \ "text": "You are a very professional document summarization specialist. Please summarize the given document" }, { - "type": "image_url", - "image_url": "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf" # 👈 PDF + "type": "file", + "file": { + "file_id": "gs://cloud-samples-data/generative-ai/pdf/2403.05530.pdf", + "format": "application/pdf" # OPTIONAL + } } } ] @@ -1858,10 +1868,11 @@ curl http://0.0.0.0:4000/v1/chat/completions \ "text": "You are a very professional document summarization specialist. Please summarize the given document" }, { - "type": "image_url", - "image_url": "data:application/pdf;base64,{encoded_file}" # 👈 PDF - } - } + "type": "file", + "file": { + "file_data": f"data:application/pdf;base64,{encoded_file}", # 👈 PDF + } + }, ] } ], @@ -1872,6 +1883,7 @@ curl http://0.0.0.0:4000/v1/chat/completions \ + ## Chat Models | Model Name | Function Call | |------------------|--------------------------------------|