scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-22 00:31:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 51

Merge pull request #4436 from BerriAI/litellm_response_cost_headers

Browse source 
fix(utils.py): return 'response_cost' in completion call
This commit is contained in:
Krish Dholakia 2024-06-27 21:33:23 -07:00 • committed by GitHub
commit efee284023
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 92 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
docs/my-website/docs/completion/token_usage.md
View file

@ -1,7 +1,21 @@
# Completion Token Usage & Cost
By default LiteLLM returns token usage in all completion requests ([See here](https://litellm.readthedocs.io/en/latest/output/))
However, we also expose some helper functions + **[NEW]** an API to calculate token usage across providers:
LiteLLM returns `response_cost` in all calls.
```python
from litellm import completion
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
mock_response="Hello world",
)
print(response._hidden_params["response_cost"])
```
LiteLLM also exposes some helper functions:
- `encode`: This encodes the text passed in, using the model-specific tokenizer. [**Jump to code**](#1-encode)
@ -23,7 +37,7 @@ However, we also expose some helper functions + **[NEW]** an API to calculate to
- `api.litellm.ai`: Live token + price count across [all supported models](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). [**Jump to code**](#10-apilitellmai)
📣 This is a community maintained list. Contributions are welcome! ❤️
📣 [This is a community maintained list](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Contributions are welcome! ❤️
## Example Usage

12
docs/my-website/docs/proxy/cost_tracking.md
View file

@ -114,6 +114,14 @@ print(response)
**Step3 - Verify Spend Tracked**
That's IT. Now Verify your spend was tracked
<Tabs>
<TabItem value="curl" label="Response Headers">
<Image img={require('../../img/response_cost_img.png')} />
</TabItem>
<TabItem value="db" label="DB + UI">
The following spend gets tracked in Table `LiteLLM_SpendLogs`
```json
@ -144,6 +152,10 @@ Use the `/global/spend/report` endpoint to get daily spend report per
- team
- customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm)
</TabItem>
</Tabs>
<Tabs>
<TabItem value="per team" label="Spend Per Team">

BIN
docs/my-website/img/response_cost_img.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 142 KiB

7
litellm/cost_calculator.py
View file

@ -572,7 +572,6 @@ def completion_cost(
completion_string = litellm.utils.get_response_string(
response_obj=completion_response
)
completion_characters = litellm.utils._count_characters(
text=completion_string
)
@ -610,7 +609,7 @@ def response_cost_calculator(
TextCompletionResponse,
],
model: str,
custom_llm_provider: str,
custom_llm_provider: Optional[str],
call_type: Literal[
"embedding",
"aembedding",
@ -632,6 +631,10 @@ def response_cost_calculator(
base_model: Optional[str] = None,
custom_pricing: Optional[bool] = None,
) -> Optional[float]:
"""
Returns
- float or None: cost of response OR none if error.
"""
try:
response_cost: float = 0.0
if cache_hit is not None and cache_hit is True:

16
litellm/proxy/proxy_server.py
View file

@ -433,6 +433,7 @@ def get_custom_headers(
api_base: Optional[str] = None,
version: Optional[str] = None,
model_region: Optional[str] = None,
response_cost: Optional[Union[float, str]] = None,
fastest_response_batch_completion: Optional[bool] = None,
**kwargs,
) -> dict:
@ -443,6 +444,7 @@ def get_custom_headers(
"x-litellm-model-api-base": api_base,
"x-litellm-version": version,
"x-litellm-model-region": model_region,
"x-litellm-response-cost": str(response_cost),
"x-litellm-key-tpm-limit": str(user_api_key_dict.tpm_limit),
"x-litellm-key-rpm-limit": str(user_api_key_dict.rpm_limit),
"x-litellm-fastest_response_batch_completion": (
@ -3048,6 +3050,7 @@ async def chat_completion(
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
fastest_response_batch_completion = hidden_params.get(
"fastest_response_batch_completion", None
)
@ -3066,6 +3069,7 @@ async def chat_completion(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
fastest_response_batch_completion=fastest_response_batch_completion,
)
@ -3095,6 +3099,7 @@ async def chat_completion(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
fastest_response_batch_completion=fastest_response_batch_completion,
**additional_headers,
@ -3290,6 +3295,7 @@ async def completion(
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
### ALERTING ###
data["litellm_status"] = "success" # used for alerting
@ -3304,6 +3310,7 @@ async def completion(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
)
selected_data_generator = select_data_generator(
response=response,
@ -3323,6 +3330,7 @@ async def completion(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
)
)
@ -3527,6 +3535,7 @@ async def embeddings(
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
fastapi_response.headers.update(
get_custom_headers(
@ -3535,6 +3544,7 @@ async def embeddings(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
)
)
@ -3676,6 +3686,7 @@ async def image_generation(
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
fastapi_response.headers.update(
get_custom_headers(
@ -3684,6 +3695,7 @@ async def image_generation(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
)
)
@ -3812,6 +3824,7 @@ async def audio_speech(
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
# Printing each chunk size
async def generate(_response: HttpxBinaryResponseContent):
@ -3825,6 +3838,7 @@ async def audio_speech(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
fastest_response_batch_completion=None,
)
@ -3976,6 +3990,7 @@ async def audio_transcriptions(
model_id = hidden_params.get("model_id", None) or ""
cache_key = hidden_params.get("cache_key", None) or ""
api_base = hidden_params.get("api_base", None) or ""
response_cost = hidden_params.get("response_cost", None) or ""
fastapi_response.headers.update(
get_custom_headers(
@ -3984,6 +3999,7 @@ async def audio_transcriptions(
cache_key=cache_key,
api_base=api_base,
version=version,
response_cost=response_cost,
model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
)
)

21
litellm/tests/test_completion_cost.py
View file

@ -712,9 +712,30 @@ def test_vertex_ai_claude_completion_cost():
assert cost == predicted_cost
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
async def test_completion_cost_hidden_params(sync_mode):
if sync_mode:
response = litellm.completion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
mock_response="Hello world",
)
else:
response = await litellm.acompletion(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": "Hey, how's it going?"}],
mock_response="Hello world",
)
assert "response_cost" in response._hidden_params
assert isinstance(response._hidden_params["response_cost"], float)
def test_vertex_ai_gemini_predict_cost():
model = "gemini-1.5-flash"
messages = [{"role": "user", "content": "Hey, hows it going???"}]
predictive_cost = completion_cost(model=model, messages=messages)
assert predictive_cost > 0

22
litellm/utils.py
View file

@ -899,6 +899,17 @@ def client(original_function):
model=model,
optional_params=getattr(logging_obj, "optional_params", {}),
)
result._hidden_params["response_cost"] = (
litellm.response_cost_calculator(
response_object=result,
model=getattr(logging_obj, "model", ""),
custom_llm_provider=getattr(
logging_obj, "custom_llm_provider", None
),
call_type=getattr(logging_obj, "call_type", "completion"),
optional_params=getattr(logging_obj, "optional_params", {}),
)
)
result._response_ms = (
end_time - start_time
).total_seconds() * 1000 # return response latency in ms like openai
@ -1292,6 +1303,17 @@ def client(original_function):
model=model,
optional_params=kwargs,
)
result._hidden_params["response_cost"] = (
litellm.response_cost_calculator(
response_object=result,
model=getattr(logging_obj, "model", ""),
custom_llm_provider=getattr(
logging_obj, "custom_llm_provider", None
),
call_type=getattr(logging_obj, "call_type", "completion"),
optional_params=getattr(logging_obj, "optional_params", {}),
)
)
if (
isinstance(result, ModelResponse)
or isinstance(result, EmbeddingResponse)