From f6546076b0e8e38939ea368fc33d898a76780509 Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrishdholakia@gmail.com>
Date: Wed, 6 Dec 2023 11:06:01 -0800
Subject: [PATCH] docs(quick_start.md): add docs on calling openai-compatible
 endpoint on proxy

---
 docs/my-website/docs/proxy/quick_start.md     | 28 +++++++++++++++----
 .../tests/test_provider_specific_config.py    |  4 +--
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/docs/my-website/docs/proxy/quick_start.md b/docs/my-website/docs/proxy/quick_start.md
index 3b5426b4cd9..d74de240d6b 100644
--- a/docs/my-website/docs/proxy/quick_start.md
+++ b/docs/my-website/docs/proxy/quick_start.md
@@ -144,7 +144,7 @@ $ litellm --model azure/my-deployment-name
 ```
 
 </TabItem>
-<TabItem value="openai-proxy" label="OpenAI">
+<TabItem value="openai" label="OpenAI">
 
 ```shell
 $ export OPENAI_API_KEY=my-api-key
@@ -154,13 +154,23 @@ $ export OPENAI_API_KEY=my-api-key
 $ litellm --model gpt-3.5-turbo
 ```
 </TabItem>
+<TabItem value="openai-proxy" label="OpenAI Compatible Endpoint">
+
+```shell
+$ export OPENAI_API_KEY=my-api-key
+```
+
+```shell
+$ litellm --model openai/<your model name> --api_base <your-api-base> # e.g. http://0.0.0.0:3000
+```
+</TabItem>
 <TabItem value="huggingface" label="Huggingface (TGI) Deployed">
 
 ```shell
 $ export HUGGINGFACE_API_KEY=my-api-key #[OPTIONAL]
 ```
 ```shell
-$ litellm --model huggingface/<your model name> --api_base https://k58ory32yinf1ly0.us-east-1.aws.endpoints.huggingface.cloud
+$ litellm --model huggingface/<your model name> --api_base <your-api-base> # e.g. http://0.0.0.0:3000
 ```
 
 </TabItem>
@@ -270,6 +280,8 @@ $ litellm --model command-nightly
 </Tabs>
 
 
+
+
 ## Quick Start - LiteLLM Proxy + Config.yaml
 The config allows you to create a model list and set `api_base`, `max_tokens` (all litellm params). See more details about the config [here](https://docs.litellm.ai/docs/proxy/configs)
 
@@ -277,9 +289,9 @@ The config allows you to create a model list and set `api_base`, `max_tokens` (a
 Example config
 
 ```yaml
-model_list:
-  - model_name: gpt-3.5-turbo
-    litellm_params:
+model_list: 
+  - model_name: gpt-3.5-turbo # user-facing model alias
+    litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input
       model: azure/<your-deployment-name>
       api_base: <your-azure-api-endpoint>
       api_key: <your-azure-api-key>
@@ -288,6 +300,10 @@ model_list:
       model: azure/gpt-turbo-small-ca
       api_base: https://my-endpoint-canada-berri992.openai.azure.com/
       api_key: <your-azure-api-key>
+  - model_name: vllm-model
+    litellm_params:
+      model: openai/<your-model-name>
+      api_base: <your-api-base> # e.g. http://0.0.0.0:3000
 ```
 
 ### Run proxy with config
@@ -296,6 +312,8 @@ model_list:
 litellm --config your_config.yaml
 ```
 
+[**More Info**](./configs.md)
+
 ## Quick Start Docker Image: Github Container Registry
 
 ### Pull the litellm ghcr docker image
diff --git a/litellm/tests/test_provider_specific_config.py b/litellm/tests/test_provider_specific_config.py
index e3a7aa82909..06282e23493 100644
--- a/litellm/tests/test_provider_specific_config.py
+++ b/litellm/tests/test_provider_specific_config.py
@@ -395,7 +395,7 @@ def sagemaker_test_completion():
     try:
         # OVERRIDE WITH DYNAMIC MAX TOKENS
         response_1 = litellm.completion(
-            model="sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b",
+            model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4",
             messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}],
             max_tokens=100
         )
@@ -404,7 +404,7 @@ def sagemaker_test_completion():
 
         # USE CONFIG TOKENS
         response_2 = litellm.completion(
-            model="sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b",
+            model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4",
             messages=[{ "content": "Hello, how are you? Be as verbose as possible","role": "user"}],
         )
         response_2_text = response_2.choices[0].message.content