diff --git a/.circleci/config.yml b/.circleci/config.yml
index 440343b45ea..4b762ca82bf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -40,7 +40,7 @@ jobs:
pip install "aioboto3==12.3.0"
pip install langchain
pip install lunary==0.2.5
- pip install "langfuse==2.7.3"
+ pip install "langfuse==2.27.1"
pip install numpydoc
pip install traceloop-sdk==0.0.69
pip install openai
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 00000000000..51c5789717f
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,47 @@
+
+
+
+## Title
+
+
+
+## Relevant issues
+
+
+
+## Type
+
+
+
+
+🆕 New Feature
+🐛 Bug Fix
+🧹 Refactoring
+📖 Documentation
+💻 Development Environment
+🚄 Infrastructure
+✅ Test
+
+## Changes
+
+
+
+## Testing
+
+
+
+## Notes
+
+
+
+
+
+## Pre-Submission Checklist (optional but appreciated):
+
+- [ ] I have included relevant documentation updates (stored in /docs/my-website)
+
+## OS Tests (optional but appreciated):
+
+- [ ] Tested on Windows
+- [ ] Tested on MacOS
+- [ ] Tested on Linux
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2a84048e0b1..cc41d85f145 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -7,7 +7,7 @@ repos:
rev: 7.0.0 # The version of flake8 to use
hooks:
- id: flake8
- exclude: ^litellm/tests/|^litellm/proxy/proxy_cli.py|^litellm/integrations/|^litellm/proxy/tests/
+ exclude: ^litellm/tests/|^litellm/proxy/proxy_cli.py|^litellm/proxy/tests/
additional_dependencies: [flake8-print]
files: litellm/.*\.py
- repo: local
diff --git a/README.md b/README.md
index 38a16693578..9344c0f223e 100644
--- a/README.md
+++ b/README.md
@@ -248,7 +248,7 @@ Step 2: Navigate into the project, and install dependencies:
```
cd litellm
-poetry install
+poetry install -E extra_proxy -E proxy
```
Step 3: Test your change:
diff --git a/docs/my-website/docs/completion/token_usage.md b/docs/my-website/docs/completion/token_usage.md
index 626973c57be..807ccfd91ec 100644
--- a/docs/my-website/docs/completion/token_usage.md
+++ b/docs/my-website/docs/completion/token_usage.md
@@ -1,7 +1,7 @@
# Completion Token Usage & Cost
By default LiteLLM returns token usage in all completion requests ([See here](https://litellm.readthedocs.io/en/latest/output/))
-However, we also expose 5 helper functions + **[NEW]** an API to calculate token usage across providers:
+However, we also expose some helper functions + **[NEW]** an API to calculate token usage across providers:
- `encode`: This encodes the text passed in, using the model-specific tokenizer. [**Jump to code**](#1-encode)
@@ -9,17 +9,19 @@ However, we also expose 5 helper functions + **[NEW]** an API to calculate token
- `token_counter`: This returns the number of tokens for a given input - it uses the tokenizer based on the model, and defaults to tiktoken if no model-specific tokenizer is available. [**Jump to code**](#3-token_counter)
-- `cost_per_token`: This returns the cost (in USD) for prompt (input) and completion (output) tokens. Uses the live list from `api.litellm.ai`. [**Jump to code**](#4-cost_per_token)
+- `create_pretrained_tokenizer` and `create_tokenizer`: LiteLLM provides default tokenizer support for OpenAI, Cohere, Anthropic, Llama2, and Llama3 models. If you are using a different model, you can create a custom tokenizer and pass it as `custom_tokenizer` to the `encode`, `decode`, and `token_counter` methods. [**Jump to code**](#4-create_pretrained_tokenizer-and-create_tokenizer)
-- `completion_cost`: This returns the overall cost (in USD) for a given LLM API Call. It combines `token_counter` and `cost_per_token` to return the cost for that query (counting both cost of input and output). [**Jump to code**](#5-completion_cost)
+- `cost_per_token`: This returns the cost (in USD) for prompt (input) and completion (output) tokens. Uses the live list from `api.litellm.ai`. [**Jump to code**](#5-cost_per_token)
-- `get_max_tokens`: This returns the maximum number of tokens allowed for the given model. [**Jump to code**](#6-get_max_tokens)
+- `completion_cost`: This returns the overall cost (in USD) for a given LLM API Call. It combines `token_counter` and `cost_per_token` to return the cost for that query (counting both cost of input and output). [**Jump to code**](#6-completion_cost)
-- `model_cost`: This returns a dictionary for all models, with their max_tokens, input_cost_per_token and output_cost_per_token. It uses the `api.litellm.ai` call shown below. [**Jump to code**](#7-model_cost)
+- `get_max_tokens`: This returns the maximum number of tokens allowed for the given model. [**Jump to code**](#7-get_max_tokens)
-- `register_model`: This registers new / overrides existing models (and their pricing details) in the model cost dictionary. [**Jump to code**](#8-register_model)
+- `model_cost`: This returns a dictionary for all models, with their max_tokens, input_cost_per_token and output_cost_per_token. It uses the `api.litellm.ai` call shown below. [**Jump to code**](#8-model_cost)
-- `api.litellm.ai`: Live token + price count across [all supported models](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). [**Jump to code**](#9-apilitellmai)
+- `register_model`: This registers new / overrides existing models (and their pricing details) in the model cost dictionary. [**Jump to code**](#9-register_model)
+
+- `api.litellm.ai`: Live token + price count across [all supported models](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). [**Jump to code**](#10-apilitellmai)
📣 This is a community maintained list. Contributions are welcome! ❤️
@@ -60,7 +62,24 @@ messages = [{"user": "role", "content": "Hey, how's it going"}]
print(token_counter(model="gpt-3.5-turbo", messages=messages))
```
-### 4. `cost_per_token`
+### 4. `create_pretrained_tokenizer` and `create_tokenizer`
+
+```python
+from litellm import create_pretrained_tokenizer, create_tokenizer
+
+# get tokenizer from huggingface repo
+custom_tokenizer_1 = create_pretrained_tokenizer("Xenova/llama-3-tokenizer")
+
+# use tokenizer from json file
+with open("tokenizer.json") as f:
+ json_data = json.load(f)
+
+json_str = json.dumps(json_data)
+
+custom_tokenizer_2 = create_tokenizer(json_str)
+```
+
+### 5. `cost_per_token`
```python
from litellm import cost_per_token
@@ -72,7 +91,7 @@ prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar = cost_per_toke
print(prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar)
```
-### 5. `completion_cost`
+### 6. `completion_cost`
* Input: Accepts a `litellm.completion()` response **OR** prompt + completion strings
* Output: Returns a `float` of cost for the `completion` call
@@ -99,7 +118,7 @@ cost = completion_cost(model="bedrock/anthropic.claude-v2", prompt="Hey!", compl
formatted_string = f"${float(cost):.10f}"
print(formatted_string)
```
-### 6. `get_max_tokens`
+### 7. `get_max_tokens`
Input: Accepts a model name - e.g., gpt-3.5-turbo (to get a complete list, call litellm.model_list).
Output: Returns the maximum number of tokens allowed for the given model
@@ -112,7 +131,7 @@ model = "gpt-3.5-turbo"
print(get_max_tokens(model)) # Output: 4097
```
-### 7. `model_cost`
+### 8. `model_cost`
* Output: Returns a dict object containing the max_tokens, input_cost_per_token, output_cost_per_token for all models on [community-maintained list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
@@ -122,7 +141,7 @@ from litellm import model_cost
print(model_cost) # {'gpt-3.5-turbo': {'max_tokens': 4000, 'input_cost_per_token': 1.5e-06, 'output_cost_per_token': 2e-06}, ...}
```
-### 8. `register_model`
+### 9. `register_model`
* Input: Provide EITHER a model cost dictionary or a url to a hosted json blob
* Output: Returns updated model_cost dictionary + updates litellm.model_cost with model details.
@@ -157,5 +176,3 @@ export LITELLM_LOCAL_MODEL_COST_MAP="True"
```
Note: this means you will need to upgrade to get updated pricing, and newer models.
-
-
diff --git a/docs/my-website/docs/observability/greenscale_integration.md b/docs/my-website/docs/observability/greenscale_integration.md
index 8fc2b7ea312..0dd673226e1 100644
--- a/docs/my-website/docs/observability/greenscale_integration.md
+++ b/docs/my-website/docs/observability/greenscale_integration.md
@@ -1,4 +1,4 @@
-# Greenscale Tutorial
+# Greenscale - Track LLM Spend and Responsible Usage
[Greenscale](https://greenscale.ai/) is a production monitoring platform for your LLM-powered app that provides you granular key insights into your GenAI spending and responsible usage. Greenscale only captures metadata to minimize the exposure risk of personally identifiable information (PII).
diff --git a/docs/my-website/docs/observability/openmeter.md b/docs/my-website/docs/observability/openmeter.md
new file mode 100644
index 00000000000..64d9c39d215
--- /dev/null
+++ b/docs/my-website/docs/observability/openmeter.md
@@ -0,0 +1,97 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# OpenMeter - Usage-Based Billing
+
+[OpenMeter](https://openmeter.io/) is an Open Source Usage-Based Billing solution for AI/Cloud applications. It integrates with Stripe for easy billing.
+
+
0?a=a.charAt(0)+"."+a.slice(1)+x(r):i>1&&(a=a.charAt(0)+"."+a.slice(1)),a=a+(o<0?"e":"e+")+o):o<0?(a="0."+x(-o-1)+a,n&&(r=n-i)>0&&(a+=x(r))):o>=i?(a+=x(o+1-i),n&&(r=n-o-1)>0&&(a=a+"."+x(r))):((r=o+1)0&&(o+1===i&&(a+="."),a+=x(r))),e.s<0?"-"+a:a}function I(e,t){if(e.length>t)return e.length=t,!0}function N(e){if(!e||"object"!=typeof e)throw Error(s+"Object expected");var t,n,r,o=["precision",1,1e9,"rounding",0,8,"toExpNeg",-1/0,0,"toExpPos",0,1/0];for(t=0;t239?4:c>223?3:c>191?2:1;if(o+d<=n)switch(d){case 1:c<128&&(u=c);break;case 2:(192&(a=e[o+1]))==128&&(s=(31&c)<<6|63&a)>127&&(u=s);break;case 3:a=e[o+1],i=e[o+2],(192&a)==128&&(192&i)==128&&(s=(15&c)<<12|(63&a)<<6|63&i)>2047&&(s<55296||s>57343)&&(u=s);break;case 4:a=e[o+1],i=e[o+2],l=e[o+3],(192&a)==128&&(192&i)==128&&(192&l)==128&&(s=(15&c)<<18|(63&a)<<12|(63&i)<<6|63&l)>65535&&s<1114112&&(u=s)}null===u?(u=65533,d=1):u>65535&&(u-=65536,r.push(u>>>10&1023|55296),u=56320|1023&u),r.push(u),o+=d}return function(e){var t=e.length;if(t<=4096)return String.fromCharCode.apply(String,e);for(var n="",r=0;r>>=0,isFinite(n)?(n>>>=0,void 0===r&&(r="utf8")):(r=n,n=void 0);else throw Error("Buffer.write(string, encoding, offset[, length]) is no longer supported");var o,a,i,l,s,c,u,d,p,f,m,g,h=this.length-t;if((void 0===n||n>h)&&(n=h),e.length>0&&(n<0||t<0)||t>this.length)throw RangeError("Attempt to write outside buffer bounds");r||(r="utf8");for(var b=!1;;)switch(r){case"hex":return function(e,t,n,r){n=Number(n)||0;var o=e.length-n;r?(r=Number(r))>o&&(r=o):r=o;var a=t.length;r>a/2&&(r=a/2);for(var i=0;i