scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge remote-tracking branch 'origin/main' into explicit-args-acomplete

Browse source
This commit is contained in:
Mateo Cámara 2024-01-09 12:04:04 +01:00
commit 178a57492b
32 changed files with 1069 additions and 1244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.circleci/config.yml
View file

@ -36,7 +36,7 @@ jobs:
pip install numpydoc
pip install traceloop-sdk==0.0.69
pip install openai
pip install prisma
pip install prisma
pip install "httpx==0.24.1"
pip install "anyio==3.7.1"
pip install "asyncio==3.4.3"
@ -44,6 +44,13 @@ jobs:
paths:
- ./venv
key: v1-dependencies-{{ checksum ".circleci/requirements.txt" }}
- run:
name: Run prisma ./entrypoint.sh
command: |
set +e
chmod +x entrypoint.sh
./entrypoint.sh
set -e
- run:
name: Black Formatting
command: |

6
.github/workflows/ghcr_deploy.yml vendored
View file

@ -44,7 +44,7 @@ jobs:
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }} # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
tags: ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, latest # if a tag is provided, use that, otherwise use the release tag, and if neither is available, use 'latest'
labels: ${{ steps.meta.outputs.labels }}
build-and-push-image-alpine:
runs-on: ubuntu-latest
@ -74,7 +74,7 @@ jobs:
context: .
dockerfile: Dockerfile.alpine
push: true
tags: ${{ steps.meta-alpine.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}
tags: ${{ steps.meta-alpine.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, latest
labels: ${{ steps.meta-alpine.outputs.labels }}
build-and-push-image-database:
runs-on: ubuntu-latest
@ -104,7 +104,7 @@ jobs:
context: .
file: Dockerfile.database
push: true
tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}
tags: ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }}, latest
labels: ${{ steps.meta-database.outputs.labels }}
release:
name: "New LiteLLM Release"

13
Dockerfile
View file

@ -34,7 +34,6 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
# Runtime stage
FROM $LITELLM_RUNTIME_IMAGE as runtime
ARG with_database
WORKDIR /app
# Copy the current directory contents into the container at /app
@ -46,12 +45,14 @@ COPY --from=builder /app/dist/*.whl .
COPY --from=builder /wheels/ /wheels/
# Install the built wheel using pip; again using a wildcard if it's the only file
RUN pip install --no-cache-dir --find-links=/wheels/ -r requirements.txt \
&& pip install *.whl \
&& rm -f *.whl
RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
RUN chmod +x entrypoint.sh
EXPOSE 4000/tcp
# Set your entrypoint and command
ENTRYPOINT ["litellm"]
CMD ["--port", "4000"]
CMD [ \
"sh", "-c", \
"if [ -n \"$DATABASE_URL\" ]; then ./entrypoint.sh; else litellm --port 4000 --num_workers 8; fi" \
]

16
README.md
View file

@ -2,7 +2,7 @@
🚅 LiteLLM
</h1>
<p align="center">
<p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, Cohere, TogetherAI, Azure, OpenAI, etc.]
<p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, etc.]
<br>
</p>
<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">OpenAI Proxy Server</a></h4>
@ -25,9 +25,9 @@
</h4>
LiteLLM manages:
- Translate inputs to provider's `completion` and `embedding` endpoints
- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
- Load-balance multiple deployments (e.g. Azure/OpenAI) - `Router` **1k+ requests/second**
- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
[**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-provider-docs)
@ -186,14 +186,15 @@ Code: https://github.com/BerriAI/litellm/tree/main/ui
<img width="1672" alt="Screenshot 2023-12-26 at 8 33 53 AM" src="https://github.com/BerriAI/litellm/assets/17561003/274254d8-c5fe-4645-9123-100045a7fb21">
## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers))
| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |
| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ |
| Provider | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |
| [openai](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [azure](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ |✅ |
| [google - vertex_ai [Gemini]](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ |
| [google - palm](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ | ✅ |
| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini) | ✅ | | ✅ | | |
| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | ✅ |
| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | ✅ |
| [cohere](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | ✅ |
@ -215,6 +216,7 @@ Code: https://github.com/BerriAI/litellm/tree/main/ui
| [voyage ai](https://docs.litellm.ai/docs/providers/voyage) | | | | | ✅ |
| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) | | | | | ✅ |
[**Read the Docs**](https://docs.litellm.ai/docs/)
## Contributing

48
docs/my-website/docs/providers/gemini.md
View file

@ -3,6 +3,7 @@
## Pre-requisites
* `pip install -q google-generativeai`
# Gemini-Pro
## Sample Usage
```python
import litellm
@ -15,14 +16,57 @@ response = completion(
)
```
# Gemini-Pro-Vision
LiteLLM Supports the following image types passed in `url`
- Images with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg
- Image in local storage - ./localimage.jpeg
## Sample Usage
```python
import os
import litellm
from dotenv import load_dotenv
# Load the environment variables from .env file
load_dotenv()
os.environ["GEMINI_API_KEY"] = os.getenv('GEMINI_API_KEY')
prompt = 'Describe the image in a few sentences.'
# Note: You can pass here the URL or Path of image directly.
image_url = 'https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg'
# Create the messages payload according to the documentation
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": prompt
},
{
"type": "image_url",
"image_url": {"url": image_url}
}
]
}
]
# Make the API call to Gemini model
response = litellm.completion(
model="gemini/gemini-pro-vision",
messages=messages,
)
# Extract the response content
content = response.get('choices', [{}])[0].get('message', {}).get('content')
# Print the result
print(content)
```
## Chat Models
| Model Name | Function Call | Required OS Variables |
|------------------|--------------------------------------|-------------------------|
| gemini-pro | `completion('gemini/gemini-pro', messages)` | `os.environ['PALM_API_KEY']` |
| gemini-pro-vision | `completion('gemini/gemini-pro-vision', messages)` | `os.environ['PALM_API_KEY']` |
| gemini-pro | `completion('gemini/gemini-pro', messages)` | `os.environ['GEMINI_API_KEY']` |
| gemini-pro-vision | `completion('gemini/gemini-pro-vision', messages)` | `os.environ['GEMINI_API_KEY']` |

207
docs/my-website/docs/proxy/deploy.md
View file

@ -3,13 +3,10 @@ import TabItem from '@theme/TabItem';
# 🐳 Docker, Deploying LiteLLM Proxy
## Dockerfile
You can find the Dockerfile to build litellm proxy [here](https://github.com/BerriAI/litellm/blob/main/Dockerfile)
## Quick Start Docker Image: Github Container Registry
## Quick Start
### Pull the litellm ghcr docker image
See the latest available ghcr docker image here:
https://github.com/berriai/litellm/pkgs/container/litellm
@ -17,12 +14,11 @@ https://github.com/berriai/litellm/pkgs/container/litellm
docker pull ghcr.io/berriai/litellm:main-latest
```
### Run the Docker Image
```shell
docker run ghcr.io/berriai/litellm:main-latest
```
#### Run the Docker Image with LiteLLM CLI args
### Run with LiteLLM CLI args
See all supported CLI args [here](https://docs.litellm.ai/docs/proxy/cli):
@ -35,8 +31,145 @@ Here's how you can run the docker image and start litellm on port 8002 with `num
```shell
docker run ghcr.io/berriai/litellm:main-latest --port 8002 --num_workers 8
```
#### Run the Docker Image using docker compose
## Deploy with Database
We maintain a [seperate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database
<Tabs>
<TabItem value="docker-deploy" label="Dockerfile">
```
docker pull docker pull ghcr.io/berriai/litellm-database:main-v1.16.20
```
```
docker run --name litellm-proxy \
-e DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname> \
-p 4000:4000 \
ghcr.io/berriai/litellm-database:main-v1.16.20
```
Your OpenAI proxy server is now running on `http://0.0.0.0:4000`.
</TabItem>
<TabItem value="kubernetes-deploy" label="Kubernetes">
### Step 1. Create deployment.yaml
```yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: litellm-deployment
spec:
replicas: 1
selector:
matchLabels:
app: litellm
template:
metadata:
labels:
app: litellm
spec:
containers:
- name: litellm-container
image: ghcr.io/berriai/litellm-database:main-v1.16.20
env:
- name: DATABASE_URL
value: postgresql://<user>:<password>@<host>:<port>/<dbname>
```
```bash
kubectl apply -f /path/to/deployment.yaml
```
### Step 2. Create service.yaml
```yaml
apiVersion: v1
kind: Service
metadata:
name: litellm-service
spec:
selector:
app: litellm
ports:
- protocol: TCP
port: 4000
targetPort: 4000
type: NodePort
```
```bash
kubectl apply -f /path/to/service.yaml
```
### Step 3. Start server
```
kubectl port-forward service/litellm-service 4000:4000
```
Your OpenAI proxy server is now running on `http://0.0.0.0:4000`.
</TabItem>
</Tabs>
## Platform-specific Guide
<Tabs>
<TabItem value="google-cloud-run" label="Google Cloud Run">
### Deploy on Google Cloud Run
**Click the button** to deploy to Google Cloud Run
[![Deploy](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/BerriAI/litellm)
#### Testing your deployed proxy
**Assuming the required keys are set as Environment Variables**
https://litellm-7yjrj3ha2q-uc.a.run.app is our example proxy, substitute it with your deployed cloud run app
```shell
curl https://litellm-7yjrj3ha2q-uc.a.run.app/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Say this is a test!"}],
"temperature": 0.7
}'
```
</TabItem>
<TabItem value="render" label="Render deploy">
### Deploy on Render https://render.com/
<iframe width="840" height="500" src="https://www.loom.com/embed/805964b3c8384b41be180a61442389a3" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
</TabItem>
<TabItem value="railway" label="Railway">
### Deploy on Railway https://railway.app
**Step 1: Click the button** to deploy to Railway
[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/S7P9sn?referralCode=t3ukrU)
**Step 2:** Set `PORT` = 4000 on Railway Environment Variables
</TabItem>
</Tabs>
## Extras
### Run with docker compose
**Step 1**
@ -80,64 +213,6 @@ Run the command `docker-compose up` or `docker compose up` as per your docker in
Your LiteLLM container should be running now on the defined port e.g. `8000`.
## Deploy with Database
#### Step 1. Save the database url in your environment
.env example: https://github.com/BerriAI/litellm/blob/main/docker/.env.example
```env
DATABASE_URL = "my-postgres-db-url"
```
#### Step 2. Build docker image with build-args
Set `with_database=true` in the docker build, to trigger the prisma logic to be run
Example build command:
```bash
docker build -t my-docker-build --build-arg with_database=true .
```
#### Step 3. Run docker image
```bash
docker run -it -p 8000:4000 my-docker-build
```
## Deploy on Render https://render.com/
<iframe width="840" height="500" src="https://www.loom.com/embed/805964b3c8384b41be180a61442389a3" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## Deploy on Google Cloud Run
**Click the button** to deploy to Google Cloud Run
[![Deploy](https://deploy.cloud.run/button.svg)](https://deploy.cloud.run/?git_repo=https://github.com/BerriAI/litellm)
#### Testing your deployed proxy
**Assuming the required keys are set as Environment Variables**
https://litellm-7yjrj3ha2q-uc.a.run.app is our example proxy, substitute it with your deployed cloud run app
```shell
curl https://litellm-7yjrj3ha2q-uc.a.run.app/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "gpt-3.5-turbo",
"messages": [{"role": "user", "content": "Say this is a test!"}],
"temperature": 0.7
}'
```
## Deploy on Railway https://railway.app
**Step 1: Click the button** to deploy to Railway
[![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/S7P9sn?referralCode=t3ukrU)
**Step 2:** Set `PORT` = 4000 on Railway Environment Variables
## LiteLLM Proxy Performance

30
docs/my-website/docs/proxy/streaming_logging.md
View file

@ -1,4 +1,4 @@
# Track Token Usage (Streaming)
# Track Token Usage
### Step 1 - Create your custom `litellm` callback class
We use `litellm.integrations.custom_logger` for this, **more details about litellm custom callbacks [here](https://docs.litellm.ai/docs/observability/custom_callback)**
@ -25,34 +25,12 @@ class MyCustomHandler(CustomLogger):
datefmt='%Y-%m-%d %H:%M:%S'
)
# check if it has collected an entire stream response
if "complete_streaming_response" in kwargs:
# for tracking streaming cost we pass the "messages" and the output_text to litellm.completion_cost
completion_response=kwargs["complete_streaming_response"]
input_text = kwargs["messages"]
output_text = completion_response["choices"][0]["message"]["content"]
response_cost = litellm.completion_cost(
model = kwargs["model"],
messages = input_text,
completion=output_text
)
print("streaming response_cost", response_cost)
logging.info(f"Model {kwargs['model']} Cost: ${response_cost:.8f}")
# for non streaming responses
else:
# we pass the completion_response obj
if kwargs["stream"] != True:
response_cost = litellm.completion_cost(completion_response=completion_response)
print("regular response_cost", response_cost)
logging.info(f"Model {completion_response.model} Cost: ${response_cost:.8f}")
response_cost = litellm.completion_cost(completion_response=completion_response)
print("regular response_cost", response_cost)
logging.info(f"Model {completion_response.model} Cost: ${response_cost:.8f}")
except:
pass
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
print(f"On Async Failure")
proxy_handler_instance = MyCustomHandler()
# Set litellm.callbacks = [proxy_handler_instance] on the proxy

8
docs/my-website/docs/proxy/virtual_keys.md
View file

@ -15,6 +15,14 @@ Complete API documentation in the Swagger docs on your proxy base url (e.g. `htt
Requirements:
- Need to a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc)
- Set `DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>` in your env
(the proxy Dockerfile checks if the `DATABASE_URL` is set and then intializes the DB connection)
```shell
export DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>
```
You can then generate temporary keys by hitting the `/key/generate` endpoint.

6
docs/my-website/package-lock.json generated
View file

@ -10891,9 +10891,9 @@
}
},
"node_modules/follow-redirects": {
"version": "1.15.2",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
"integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
"version": "1.15.4",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz",
"integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==",
"funding": [
{
"type": "individual",

1483
docs/my-website/yarn.lock
View file

File diff suppressed because it is too large Load diff

21
litellm/llms/azure.py
View file

@ -268,7 +268,10 @@ class AzureChatCompletion(BaseLLM):
exception_mapping_worked = True
raise e
except Exception as e:
raise e
if hasattr(e, "status_code"):
raise AzureOpenAIError(status_code=e.status_code, message=str(e))
else:
raise AzureOpenAIError(status_code=500, message=str(e))
async def acompletion(
self,
@ -569,12 +572,10 @@ class AzureChatCompletion(BaseLLM):
exception_mapping_worked = True
raise e
except Exception as e:
if exception_mapping_worked:
raise e
if hasattr(e, "status_code"):
raise AzureOpenAIError(status_code=e.status_code, message=str(e))
else:
import traceback
raise AzureOpenAIError(status_code=500, message=traceback.format_exc())
raise AzureOpenAIError(status_code=500, message=str(e))
async def aimage_generation(
self,
@ -702,12 +703,10 @@ class AzureChatCompletion(BaseLLM):
exception_mapping_worked = True
raise e
except Exception as e:
if exception_mapping_worked:
raise e
if hasattr(e, "status_code"):
raise AzureOpenAIError(status_code=e.status_code, message=str(e))
else:
import traceback
raise AzureOpenAIError(status_code=500, message=traceback.format_exc())
raise AzureOpenAIError(status_code=500, message=str(e))
async def ahealth_check(
self,

9
litellm/llms/ollama.py
View file

@ -179,10 +179,7 @@ def get_ollama_response(
elif optional_params.get("stream", False) == True:
return ollama_completion_stream(url=url, data=data, logging_obj=logging_obj)
response = requests.post(
url=f"{url}",
json=data,
)
response = requests.post(url=f"{url}", json=data, timeout=litellm.request_timeout)
if response.status_code != 200:
raise OllamaError(status_code=response.status_code, message=response.text)
@ -217,7 +214,7 @@ def get_ollama_response(
model_response["choices"][0]["message"]["content"] = response_json["response"]
model_response["created"] = int(time.time())
model_response["model"] = "ollama/" + model
prompt_tokens = response_json["prompt_eval_count"] # type: ignore
prompt_tokens = response_json.get("prompt_eval_count", len(encoding.encode(prompt))) # type: ignore
completion_tokens = response_json["eval_count"]
model_response["usage"] = litellm.Usage(
prompt_tokens=prompt_tokens,
@ -318,7 +315,7 @@ async def ollama_acompletion(url, data, model_response, encoding, logging_obj):
]
model_response["created"] = int(time.time())
model_response["model"] = "ollama/" + data["model"]
prompt_tokens = response_json["prompt_eval_count"] # type: ignore
prompt_tokens = response_json.get("prompt_eval_count", len(encoding.encode(data["prompt"]))) # type: ignore
completion_tokens = response_json["eval_count"]
model_response["usage"] = litellm.Usage(
prompt_tokens=prompt_tokens,

23
litellm/llms/openai.py
View file

@ -346,7 +346,10 @@ class OpenAIChatCompletion(BaseLLM):
exception_mapping_worked = True
raise e
except Exception as e:
raise e
if hasattr(e, "status_code"):
raise OpenAIError(status_code=e.status_code, message=str(e))
else:
raise OpenAIError(status_code=500, message=str(e))
async def acompletion(
self,
@ -500,6 +503,8 @@ class OpenAIChatCompletion(BaseLLM):
else:
if type(e).__name__ == "ReadTimeout":
raise OpenAIError(status_code=408, message=f"{type(e).__name__}")
elif hasattr(e, "status_code"):
raise OpenAIError(status_code=e.status_code, message=str(e))
else:
raise OpenAIError(status_code=500, message=f"{str(e)}")
@ -603,12 +608,10 @@ class OpenAIChatCompletion(BaseLLM):
exception_mapping_worked = True
raise e
except Exception as e:
if exception_mapping_worked:
raise e
if hasattr(e, "status_code"):
raise OpenAIError(status_code=e.status_code, message=str(e))
else:
import traceback
raise OpenAIError(status_code=500, message=traceback.format_exc())
raise OpenAIError(status_code=500, message=str(e))
async def aimage_generation(
self,
@ -716,12 +719,10 @@ class OpenAIChatCompletion(BaseLLM):
exception_mapping_worked = True
raise e
except Exception as e:
if exception_mapping_worked:
raise e
if hasattr(e, "status_code"):
raise OpenAIError(status_code=e.status_code, message=str(e))
else:
import traceback
raise OpenAIError(status_code=500, message=traceback.format_exc())
raise OpenAIError(status_code=500, message=str(e))
async def ahealth_check(
self,

15
litellm/proxy/proxy_server.py
View file

@ -311,6 +311,7 @@ def prisma_setup(database_url: Optional[str]):
database_url=database_url, proxy_logging_obj=proxy_logging_obj
)
except Exception as e:
raise e
verbose_proxy_logger.debug(
f"Error when initializing prisma, Ensure you run pip install prisma {str(e)}"
)
@ -510,10 +511,9 @@ class ProxyConfig:
def is_yaml(self, config_file_path: str) -> bool:
if not os.path.isfile(config_file_path):
return False
_, file_extension = os.path.splitext(config_file_path)
return file_extension.lower() == '.yaml' or file_extension.lower() == '.yml'
return file_extension.lower() == ".yaml" or file_extension.lower() == ".yml"
async def get_config(self, config_file_path: Optional[str] = None) -> dict:
global prisma_client, user_config_file_path
@ -776,7 +776,6 @@ class ProxyConfig:
verbose_proxy_logger.debug(f"GOING INTO LITELLM.GET_SECRET!")
database_url = litellm.get_secret(database_url)
verbose_proxy_logger.debug(f"RETRIEVED DB URL: {database_url}")
prisma_setup(database_url=database_url)
## COST TRACKING ##
cost_tracking()
### MASTER KEY ###
@ -1156,7 +1155,7 @@ async def startup_event():
### CONNECT TO DB ###
# check if DATABASE_URL in environment - load from there
if os.getenv("DATABASE_URL", None) is not None and prisma_client is None:
if prisma_client is None:
prisma_setup(database_url=os.getenv("DATABASE_URL"))
### LOAD CONFIG ###
@ -1169,7 +1168,9 @@ async def startup_event():
llm_router,
llm_model_list,
general_settings,
) = await proxy_config.load_config(router=llm_router, config_file_path=worker_config)
) = await proxy_config.load_config(
router=llm_router, config_file_path=worker_config
)
else:
await initialize(**worker_config)
else:
@ -1184,7 +1185,7 @@ async def startup_event():
) # start the background health check coroutine.
verbose_proxy_logger.debug(f"prisma client - {prisma_client}")
if prisma_client:
if prisma_client is not None:
await prisma_client.connect()
if prisma_client is not None and master_key is not None:

3
litellm/proxy/utils.py
View file

@ -555,7 +555,8 @@ class PrismaClient:
)
async def connect(self):
try:
await self.db.connect()
if self.db.is_connected() == False:
await self.db.connect()
except Exception as e:
asyncio.create_task(
self.proxy_logging_obj.failure_handler(original_exception=e)

29
litellm/router_strategy/lowest_latency.py
View file

@ -50,9 +50,12 @@ class LowestLatencyLoggingHandler(CustomLogger):
## Latency
request_count_dict = self.router_cache.get_cache(key=latency_key) or {}
request_count_dict[id] = response_ms
if id in request_count_dict and isinstance(request_count_dict[id], list):
request_count_dict[id] = request_count_dict[id].append(response_ms)
else:
request_count_dict[id] = [response_ms]
self.router_cache.set_cache(key=latency_key, value=request_count_dict)
self.router_cache.set_cache(key=latency_key, value=request_count_dict, ttl=self.default_cache_time_seconds) # reset map within window
### TESTING ###
if self.test_flag:
@ -90,9 +93,12 @@ class LowestLatencyLoggingHandler(CustomLogger):
## Latency
request_count_dict = self.router_cache.get_cache(key=latency_key) or {}
request_count_dict[id] = response_ms
if id in request_count_dict and isinstance(request_count_dict[id], list):
request_count_dict[id] = request_count_dict[id] + [response_ms]
else:
request_count_dict[id] = [response_ms]
self.router_cache.set_cache(key=latency_key, value=request_count_dict)
self.router_cache.set_cache(key=latency_key, value=request_count_dict, ttl=self.default_cache_time_seconds) # reset map within window
### TESTING ###
if self.test_flag:
@ -123,7 +129,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
for d in healthy_deployments:
## if healthy deployment not yet used
if d["model_info"]["id"] not in all_deployments:
all_deployments[d["model_info"]["id"]] = 0
all_deployments[d["model_info"]["id"]] = [0]
for item, item_latency in all_deployments.items():
## get the item from model list
@ -134,10 +140,15 @@ class LowestLatencyLoggingHandler(CustomLogger):
if _deployment is None:
continue # skip to next one
if isinstance(item_latency, timedelta):
item_latency = float(item_latency.total_seconds())
# get average latency
total = 0.0
for _call_latency in item_latency:
if isinstance(_call_latency, timedelta):
total += float(_call_latency.total_seconds())
elif isinstance(_call_latency, float):
total += _call_latency
item_latency = total/len(item_latency)
if item_latency == 0:
deployment = _deployment
break

2
litellm/tests/example_config_yaml/aliases_config.yaml
View file

@ -1,5 +1,5 @@
model_list:
- model_name: text-davinci-003
- model_name: gpt-3.5-turbo-instruct
litellm_params:
model: ollama/zephyr
- model_name: gpt-4

44
litellm/tests/test_completion.py
View file

@ -744,18 +744,19 @@ def test_completion_openai_litellm_key():
def test_completion_ollama_hosted():
try:
litellm.request_timeout = 20 # give ollama 20 seconds to response
litellm.set_verbose = True
response = completion(
model="ollama/phi",
messages=messages,
max_tokens=10,
num_retries=3,
timeout=90,
max_tokens=2,
api_base="https://test-ollama-endpoint.onrender.com",
)
# Add any assertions here to check the response
print(response)
except Timeout as e:
except openai.APITimeoutError as e:
print("got a timeout error. Passed ! ")
litellm.request_timeout = None
pass
except Exception as e:
pytest.fail(f"Error occurred: {e}")
@ -1130,10 +1131,11 @@ def test_completion_azure_deployment_id():
# test_completion_anthropic_openai_proxy()
@pytest.mark.skip(reason="replicate endpoints take +2 mins just for this request")
def test_completion_replicate_vicuna():
print("TESTING REPLICATE")
litellm.set_verbose = True
model_name = "replicate/vicuna-13b:6282abe6a492de4145d7bb601023762212f9ddbbe78278bd6771c8b3b2f2a13b"
model_name = "replicate/meta/llama-2-7b-chat:f1d50bb24186c52daae319ca8366e53debdaa9e0ae7ff976e918df752732ccc4"
try:
response = completion(
model=model_name,
@ -1143,7 +1145,7 @@ def test_completion_replicate_vicuna():
repetition_penalty=1,
min_tokens=1,
seed=-1,
max_tokens=20,
max_tokens=2,
)
print(response)
# Add any assertions here to check the response
@ -1156,36 +1158,6 @@ def test_completion_replicate_vicuna():
# test_completion_replicate_vicuna()
# commenting out - flaky test
# def test_completion_replicate_llama2_stream():
# litellm.set_verbose=False
# model_name = "replicate/meta/llama-2-7b-chat:13c3cdee13ee059ab779f0291d29054dab00a47dad8261375654de5540165fb0"
# try:
# response = completion(
# model=model_name,
# messages=[
# {
# "role": "user",
# "content": "what is yc write 1 paragraph",
# }
# ],
# stream=True,
# max_tokens=20,
# num_retries=3
# )
# print(f"response: {response}")
# # Add any assertions here to check the response
# complete_response = ""
# for i, chunk in enumerate(response):
# complete_response += chunk.choices[0].delta["content"]
# # if i == 0:
# # assert len(chunk.choices[0].delta["content"]) > 2
# # print(chunk)
# assert len(complete_response) > 5
# print(f"complete_response: {complete_response}")
# except Exception as e:
# pytest.fail(f"Error occurred: {e}")
# test_completion_replicate_llama2_stream()
def test_replicate_custom_prompt_dict():

2
litellm/tests/test_configs/test_config.yaml
View file

@ -1,5 +1,5 @@
general_settings:
database_url: os.environ/PROXY_DATABASE_URL
database_url: os.environ/DATABASE_URL
master_key: os.environ/PROXY_MASTER_KEY
litellm_settings:
drop_params: true

2
litellm/tests/test_custom_logger.py
View file

@ -183,7 +183,7 @@ def test_azure_completion_stream():
# checks if the model response available in the async + stream callbacks is equal to the received response
customHandler2 = MyCustomHandler()
litellm.callbacks = [customHandler2]
litellm.set_verbose = True
litellm.set_verbose = False
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{

54
litellm/tests/test_deployed_proxy_keygen.py Normal file
View file

@ -0,0 +1,54 @@
import sys, os, time
import traceback
from dotenv import load_dotenv
load_dotenv()
import os, io
# this file is to test litellm/proxy
sys.path.insert(
0, os.path.abspath("../..")
) # Adds the parent directory to the system path
import pytest, logging, requests
import litellm
from litellm import embedding, completion, completion_cost, Timeout
from litellm import RateLimitError
def test_add_new_key():
try:
# Your test data
test_data = {
"models": ["gpt-3.5-turbo", "gpt-4", "claude-2", "azure-model"],
"aliases": {"mistral-7b": "gpt-3.5-turbo"},
"duration": "20m",
}
print("testing proxy server")
# Your bearer token
token = os.getenv("PROXY_MASTER_KEY")
headers = {"Authorization": f"Bearer {token}"}
staging_endpoint = "https://litellm-litellm-pr-1376.up.railway.app"
main_endpoint = "https://litellm-staging.up.railway.app"
# Your bearer token
token = os.getenv("PROXY_MASTER_KEY")
headers = {"Authorization": f"Bearer {token}"}
# Make a request to the staging endpoint
response = requests.post(
main_endpoint + "/key/generate", json=test_data, headers=headers
)
print(f"response: {response.text}")
assert response.status_code == 200
result = response.json()
except Exception as e:
print(traceback.format_exc())
pytest.fail(f"An error occurred {e}")
# test_add_new_key()

87
litellm/tests/test_image_generation.py
View file

@ -19,57 +19,80 @@ import litellm
def test_image_generation_openai():
litellm.set_verbose = True
response = litellm.image_generation(
prompt="A cute baby sea otter", model="dall-e-3"
)
print(f"response: {response}")
assert len(response.data) > 0
try:
litellm.set_verbose = True
response = litellm.image_generation(
prompt="A cute baby sea otter", model="dall-e-3"
)
print(f"response: {response}")
assert len(response.data) > 0
except litellm.RateLimitError as e:
pass
except Exception as e:
pytest.fail(f"An exception occurred - {str(e)}")
# test_image_generation_openai()
def test_image_generation_azure():
response = litellm.image_generation(
prompt="A cute baby sea otter", model="azure/", api_version="2023-06-01-preview"
)
print(f"response: {response}")
assert len(response.data) > 0
try:
response = litellm.image_generation(
prompt="A cute baby sea otter", model="azure/", api_version="2023-06-01-preview"
)
print(f"response: {response}")
assert len(response.data) > 0
except litellm.RateLimitError as e:
pass
except Exception as e:
pytest.fail(f"An exception occurred - {str(e)}")
# test_image_generation_azure()
def test_image_generation_azure_dall_e_3():
litellm.set_verbose = True
response = litellm.image_generation(
prompt="A cute baby sea otter",
model="azure/dall-e-3-test",
api_version="2023-12-01-preview",
api_base=os.getenv("AZURE_SWEDEN_API_BASE"),
api_key=os.getenv("AZURE_SWEDEN_API_KEY"),
)
print(f"response: {response}")
assert len(response.data) > 0
try:
litellm.set_verbose = True
response = litellm.image_generation(
prompt="A cute baby sea otter",
model="azure/dall-e-3-test",
api_version="2023-12-01-preview",
api_base=os.getenv("AZURE_SWEDEN_API_BASE"),
api_key=os.getenv("AZURE_SWEDEN_API_KEY"),
)
print(f"response: {response}")
assert len(response.data) > 0
except litellm.RateLimitError as e:
pass
except Exception as e:
pytest.fail(f"An exception occurred - {str(e)}")
# test_image_generation_azure_dall_e_3()
@pytest.mark.asyncio
async def test_async_image_generation_openai():
response = litellm.image_generation(
prompt="A cute baby sea otter", model="dall-e-3"
)
print(f"response: {response}")
assert len(response.data) > 0
try:
response = litellm.image_generation(
prompt="A cute baby sea otter", model="dall-e-3"
)
print(f"response: {response}")
assert len(response.data) > 0
except litellm.RateLimitError as e:
pass
except Exception as e:
pytest.fail(f"An exception occurred - {str(e)}")
# asyncio.run(test_async_image_generation_openai())
@pytest.mark.asyncio
async def test_async_image_generation_azure():
response = await litellm.aimage_generation(
prompt="A cute baby sea otter", model="azure/dall-e-3-test"
)
print(f"response: {response}")
try:
response = await litellm.aimage_generation(
prompt="A cute baby sea otter", model="azure/dall-e-3-test"
)
print(f"response: {response}")
except litellm.RateLimitError as e:
pass
except Exception as e:
pytest.fail(f"An exception occurred - {str(e)}")

2
litellm/tests/test_lowest_latency_routing.py
View file

@ -49,7 +49,7 @@ def test_latency_updated():
end_time=end_time,
)
latency_key = f"{model_group}_latency_map"
assert end_time - start_time == test_cache.get_cache(key=latency_key)[deployment_id]
assert end_time - start_time == test_cache.get_cache(key=latency_key)[deployment_id][0]
# test_tpm_rpm_updated()

6
litellm/tests/test_provider_specific_config.py
View file

@ -602,7 +602,7 @@ def openai_text_completion_test():
try:
# OVERRIDE WITH DYNAMIC MAX TOKENS
response_1 = litellm.completion(
model="text-davinci-003",
model="gpt-3.5-turbo-instruct",
messages=[
{
"content": "Hello, how are you? Be as verbose as possible",
@ -616,7 +616,7 @@ def openai_text_completion_test():
# USE CONFIG TOKENS
response_2 = litellm.completion(
model="text-davinci-003",
model="gpt-3.5-turbo-instruct",
messages=[
{
"content": "Hello, how are you? Be as verbose as possible",
@ -630,7 +630,7 @@ def openai_text_completion_test():
assert len(response_2_text) < len(response_1_text)
response_3 = litellm.completion(
model="text-davinci-003",
model="gpt-3.5-turbo-instruct",
messages=[{"content": "Hello, how are you?", "role": "user"}],
n=2,
)

5
litellm/tests/test_proxy_startup.py
View file

@ -19,7 +19,8 @@ from litellm.proxy.proxy_server import (
save_worker_config,
initialize,
startup_event,
llm_model_list
llm_model_list,
shutdown_event
)
def test_proxy_gunicorn_startup_direct_config():
@ -36,6 +37,7 @@ def test_proxy_gunicorn_startup_direct_config():
config_fp = f"{filepath}/test_configs/test_config_no_auth.yaml"
os.environ["WORKER_CONFIG"] = config_fp
asyncio.run(startup_event())
asyncio.run(shutdown_event())
except Exception as e:
if "Already connected to the query engine" in str(e):
pass
@ -51,6 +53,7 @@ def test_proxy_gunicorn_startup_config_dict():
worker_config = {"config": config_fp}
os.environ["WORKER_CONFIG"] = json.dumps(worker_config)
asyncio.run(startup_event())
asyncio.run(shutdown_event())
except Exception as e:
if "Already connected to the query engine" in str(e):
pass

7
litellm/tests/test_router.py
View file

@ -456,8 +456,11 @@ async def test_aimg_gen_on_router():
router.reset()
except Exception as e:
traceback.print_exc()
pytest.fail(f"Error occurred: {e}")
if "Your task failed as a result of our safety system." in str(e):
pass
else:
traceback.print_exc()
pytest.fail(f"Error occurred: {e}")
# asyncio.run(test_aimg_gen_on_router())

2
litellm/tests/test_streaming.py
View file

@ -307,7 +307,7 @@ def test_completion_ollama_hosted_stream():
messages=messages,
max_tokens=10,
num_retries=3,
timeout=90,
timeout=20,
api_base="https://test-ollama-endpoint.onrender.com",
stream=True,
)

21
litellm/tests/test_text_completion.py
View file

@ -2682,7 +2682,7 @@ def test_completion_openai_prompt():
try:
print("\n text 003 test\n")
response = text_completion(
model="text-davinci-003", prompt="What's the weather in SF?"
model="gpt-3.5-turbo-instruct", prompt="What's the weather in SF?"
)
print(response)
response_str = response["choices"][0]["text"]
@ -2700,7 +2700,7 @@ def test_completion_openai_engine_and_model():
print("\n text 003 test\n")
litellm.set_verbose = True
response = text_completion(
model="text-davinci-003",
model="gpt-3.5-turbo-instruct",
engine="anything",
prompt="What's the weather in SF?",
max_tokens=5,
@ -2721,7 +2721,9 @@ def test_completion_openai_engine():
print("\n text 003 test\n")
litellm.set_verbose = True
response = text_completion(
engine="text-davinci-003", prompt="What's the weather in SF?", max_tokens=5
engine="gpt-3.5-turbo-instruct",
prompt="What's the weather in SF?",
max_tokens=5,
)
print(response)
response_str = response["choices"][0]["text"]
@ -2754,14 +2756,13 @@ def test_completion_chatgpt_prompt():
def test_text_completion_basic():
try:
print("\n test 003 with echo and logprobs \n")
print("\n test 003 with logprobs \n")
litellm.set_verbose = False
response = text_completion(
model="text-davinci-003",
model="gpt-3.5-turbo-instruct",
prompt="good morning",
max_tokens=10,
logprobs=10,
echo=True,
)
print(response)
print(response.choices)
@ -2779,7 +2780,7 @@ def test_completion_text_003_prompt_array():
try:
litellm.set_verbose = False
response = text_completion(
model="text-davinci-003",
model="gpt-3.5-turbo-instruct",
prompt=token_prompt, # token prompt is a 2d list
)
print("\n\n response")
@ -2832,6 +2833,8 @@ def test_completion_hf_prompt_array():
assert len(response.choices) == 2
# response_str = response["choices"][0]["text"]
except Exception as e:
if "is currently loading" in str(e):
pass
pytest.fail(f"Error occurred: {e}")
@ -2857,7 +2860,7 @@ def test_text_completion_stream():
# async def test_text_completion_async_stream():
# try:
# response = await atext_completion(
# model="text-completion-openai/text-davinci-003",
# model="text-completion-openai/gpt-3.5-turbo-instruct",
# prompt="good morning",
# stream=True,
# max_tokens=10,
@ -2938,7 +2941,7 @@ async def test_async_text_completion_chat_model_stream():
)
num_finish_reason = 0
chunks = []
chunks = []
async for chunk in response:
print(chunk)
chunks.append(chunk)

27
litellm/tests/test_timeout.py
View file

@ -80,7 +80,7 @@ def test_hanging_request_azure():
)
test_hanging_request_azure()
# test_hanging_request_azure()
def test_hanging_request_openai():
@ -156,3 +156,28 @@ def test_timeout_streaming():
# test_timeout_streaming()
def test_timeout_ollama():
# this Will Raise a timeout
import litellm
litellm.set_verbose = True
try:
litellm.request_timeout = 0.1
litellm.set_verbose = True
response = litellm.completion(
model="ollama/phi",
messages=[{"role": "user", "content": "hello, what llm are u"}],
max_tokens=1,
api_base="https://test-ollama-endpoint.onrender.com",
)
# Add any assertions here to check the response
litellm.request_timeout = None
print(response)
except openai.APITimeoutError as e:
print("got a timeout error! Passed ! ")
pass
# test_timeout_ollama()

34
litellm/utils.py
View file

@ -558,7 +558,7 @@ class TextChoices(OpenAIObject):
def __setitem__(self, key, value):
# Allow dictionary-style assignment of attributes
setattr(self, key, value)
def json(self, **kwargs):
try:
return self.model_dump() # noqa
@ -737,7 +737,9 @@ class Logging:
f"Invalid call_type {call_type}. Allowed values: {allowed_values}"
)
if messages is not None and isinstance(messages, str):
messages = [{"role": "user", "content": messages}] # convert text completion input to the chat completion format
messages = [
{"role": "user", "content": messages}
] # convert text completion input to the chat completion format
self.model = model
self.messages = messages
self.stream = stream
@ -4002,7 +4004,8 @@ def get_llm_provider(
if (
model.split("/", 1)[0] in litellm.provider_list
and model.split("/", 1)[0] not in litellm.model_list
and len(model.split("/")) > 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351
and len(model.split("/"))
> 1 # handle edge case where user passes in `litellm --model mistral` https://github.com/BerriAI/litellm/issues/1351
):
custom_llm_provider = model.split("/", 1)[0]
model = model.split("/", 1)[1]
@ -4137,15 +4140,15 @@ def get_llm_provider(
raise e
else:
raise litellm.exceptions.BadRequestError( # type: ignore
message=f"GetLLMProvider Exception - {str(e)}\n\noriginal model: {model}",
model=model,
response=httpx.Response(
status_code=400,
content=error_str,
request=httpx.request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
),
llm_provider="",
)
message=f"GetLLMProvider Exception - {str(e)}\n\noriginal model: {model}",
model=model,
response=httpx.Response(
status_code=400,
content=error_str,
request=httpx.request(method="completion", url="https://github.com/BerriAI/litellm"), # type: ignore
),
llm_provider="",
)
def get_api_key(llm_provider: str, dynamic_api_key: Optional[str]):
@ -6460,6 +6463,13 @@ def exception_type(
model=model,
response=original_exception.response,
)
elif "Read timed out" in error_str:
exception_mapping_worked = True
raise Timeout(
message=f"OllamaException: {original_exception}",
llm_provider="ollama",
model=model,
)
elif custom_llm_provider == "vllm":
if hasattr(original_exception, "status_code"):
if original_exception.status_code == 0:

88
model_prices_and_context_window.json
View file

@ -1,6 +1,8 @@
{
"gpt-4": {
"max_tokens": 8192,
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "openai",
@ -8,6 +10,8 @@
},
"gpt-4-0314": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "openai",
@ -15,6 +19,8 @@
},
"gpt-4-0613": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "openai",
@ -22,6 +28,8 @@
},
"gpt-4-32k": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00006,
"output_cost_per_token": 0.00012,
"litellm_provider": "openai",
@ -29,6 +37,8 @@
},
"gpt-4-32k-0314": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00006,
"output_cost_per_token": 0.00012,
"litellm_provider": "openai",
@ -36,6 +46,8 @@
},
"gpt-4-32k-0613": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00006,
"output_cost_per_token": 0.00012,
"litellm_provider": "openai",
@ -43,6 +55,8 @@
},
"gpt-4-1106-preview": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
@ -50,6 +64,8 @@
},
"gpt-4-vision-preview": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "openai",
@ -57,6 +73,8 @@
},
"gpt-3.5-turbo": {
"max_tokens": 4097,
"max_input_tokens": 4097,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "openai",
@ -64,6 +82,8 @@
},
"gpt-3.5-turbo-0301": {
"max_tokens": 4097,
"max_input_tokens": 4097,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "openai",
@ -71,6 +91,8 @@
},
"gpt-3.5-turbo-0613": {
"max_tokens": 4097,
"max_input_tokens": 4097,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "openai",
@ -78,6 +100,8 @@
},
"gpt-3.5-turbo-1106": {
"max_tokens": 16385,
"max_input_tokens": 16385,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000010,
"output_cost_per_token": 0.0000020,
"litellm_provider": "openai",
@ -85,6 +109,8 @@
},
"gpt-3.5-turbo-16k": {
"max_tokens": 16385,
"max_input_tokens": 16385,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000004,
"litellm_provider": "openai",
@ -92,6 +118,8 @@
},
"gpt-3.5-turbo-16k-0613": {
"max_tokens": 16385,
"max_input_tokens": 16385,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000004,
"litellm_provider": "openai",
@ -99,6 +127,8 @@
},
"ft:gpt-3.5-turbo": {
"max_tokens": 4097,
"max_input_tokens": 4097,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000012,
"output_cost_per_token": 0.000016,
"litellm_provider": "openai",
@ -174,6 +204,8 @@
},
"azure/gpt-4-1106-preview": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
@ -181,6 +213,8 @@
},
"azure/gpt-4-0613": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "azure",
@ -188,6 +222,8 @@
},
"azure/gpt-4-32k-0613": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00006,
"output_cost_per_token": 0.00012,
"litellm_provider": "azure",
@ -195,6 +231,8 @@
},
"azure/gpt-4-32k": {
"max_tokens": 32768,
"max_input_tokens": 32768,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00006,
"output_cost_per_token": 0.00012,
"litellm_provider": "azure",
@ -202,6 +240,8 @@
},
"azure/gpt-4": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00006,
"litellm_provider": "azure",
@ -209,6 +249,8 @@
},
"azure/gpt-4-turbo": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
@ -216,6 +258,8 @@
},
"azure/gpt-4-turbo-vision-preview": {
"max_tokens": 128000,
"max_input_tokens": 128000,
"max_output_tokens": 4096,
"input_cost_per_token": 0.00001,
"output_cost_per_token": 0.00003,
"litellm_provider": "azure",
@ -223,6 +267,8 @@
},
"azure/gpt-35-turbo-16k-0613": {
"max_tokens": 16385,
"max_input_tokens": 16385,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000004,
"litellm_provider": "azure",
@ -230,6 +276,8 @@
},
"azure/gpt-35-turbo-1106": {
"max_tokens": 16384,
"max_input_tokens": 16384,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "azure",
@ -237,6 +285,8 @@
},
"azure/gpt-35-turbo-16k": {
"max_tokens": 16385,
"max_input_tokens": 16385,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000003,
"output_cost_per_token": 0.000004,
"litellm_provider": "azure",
@ -244,6 +294,8 @@
},
"azure/gpt-35-turbo": {
"max_tokens": 4097,
"max_input_tokens": 4097,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "azure",
@ -305,36 +357,10 @@
"litellm_provider": "azure",
"mode": "image_generation"
},
"text-davinci-003": {
"max_tokens": 4097,
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000002,
"litellm_provider": "text-completion-openai",
"mode": "completion"
},
"text-curie-001": {
"max_tokens": 2049,
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000002,
"litellm_provider": "text-completion-openai",
"mode": "completion"
},
"text-babbage-001": {
"max_tokens": 2049,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.0000004,
"litellm_provider": "text-completion-openai",
"mode": "completion"
},
"text-ada-001": {
"max_tokens": 2049,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.0000004,
"litellm_provider": "text-completion-openai",
"mode": "completion"
},
"babbage-002": {
"max_tokens": 16384,
"max_input_tokens": 16384,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000004,
"output_cost_per_token": 0.0000004,
"litellm_provider": "text-completion-openai",
@ -342,6 +368,8 @@
},
"davinci-002": {
"max_tokens": 16384,
"max_input_tokens": 16384,
"max_output_tokens": 4096,
"input_cost_per_token": 0.000002,
"output_cost_per_token": 0.000002,
"litellm_provider": "text-completion-openai",
@ -349,6 +377,8 @@
},
"gpt-3.5-turbo-instruct": {
"max_tokens": 8192,
"max_input_tokens": 8192,
"max_output_tokens": 4096,
"input_cost_per_token": 0.0000015,
"output_cost_per_token": 0.000002,
"litellm_provider": "text-completion-openai",

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.16.20"
version = "1.16.21"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT License"
@ -59,7 +59,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.16.20"
version = "1.16.21"
version_files = [
"pyproject.toml:^version"
]