diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 062e98be97b..5d976483a9e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -4,6 +4,8 @@ import time import traceback from typing import List, Literal, Optional, Tuple, Union +from pydantic import BaseModel + import litellm import litellm._logging from litellm import verbose_logger @@ -13,6 +15,9 @@ from litellm.litellm_core_utils.llm_cost_calc.google import ( from litellm.litellm_core_utils.llm_cost_calc.google import ( cost_per_token as google_cost_per_token, ) +from litellm.litellm_core_utils.llm_cost_calc.utils import _generic_cost_per_character +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS from litellm.utils import ( CallTypes, CostPerToken, @@ -62,6 +67,23 @@ def cost_per_token( ### CUSTOM PRICING ### custom_cost_per_token: Optional[CostPerToken] = None, custom_cost_per_second: Optional[float] = None, + ### CALL TYPE ### + call_type: Literal[ + "embedding", + "aembedding", + "completion", + "acompletion", + "atext_completion", + "text_completion", + "image_generation", + "aimage_generation", + "moderation", + "amoderation", + "atranscription", + "transcription", + "aspeech", + "speech", + ] = "completion", ) -> Tuple[float, float]: """ Calculates the cost per token for a given model, prompt tokens, and completion tokens. @@ -76,6 +98,7 @@ def cost_per_token( custom_llm_provider (str): The llm provider to whom the call was made (see init.py for full list) custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call. custom_cost_per_second: Optional[float]: the cost per second for the llm api call. + call_type: Optional[str]: the call type Returns: tuple: A tuple containing the cost in USD dollars for prompt tokens and completion tokens, respectively. @@ -159,6 +182,27 @@ def cost_per_token( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, ) + elif call_type == "speech" or call_type == "aspeech": + prompt_cost, completion_cost = _generic_cost_per_character( + model=model_without_prefix, + custom_llm_provider=custom_llm_provider, + prompt_characters=prompt_characters, + completion_characters=completion_characters, + custom_prompt_cost=None, + custom_completion_cost=0, + ) + if prompt_cost is None or completion_cost is None: + raise ValueError( + "cost for tts call is None. prompt_cost={}, completion_cost={}, model={}, custom_llm_provider={}, prompt_characters={}, completion_characters={}".format( + prompt_cost, + completion_cost, + model_without_prefix, + custom_llm_provider, + prompt_characters, + completion_characters, + ) + ) + return prompt_cost, completion_cost elif model in model_cost_ref: print_verbose(f"Success: model={model} in model_cost_map") print_verbose( @@ -289,7 +333,7 @@ def cost_per_token( return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar else: # if model is not in model_prices_and_context_window.json. Raise an exception-let users know - error_str = f"Model not in model_prices_and_context_window.json. You passed model={model}. Register pricing for model - https://docs.litellm.ai/docs/proxy/custom_pricing\n" + error_str = f"Model not in model_prices_and_context_window.json. You passed model={model}, custom_llm_provider={custom_llm_provider}. Register pricing for model - https://docs.litellm.ai/docs/proxy/custom_pricing\n" raise litellm.exceptions.NotFoundError( # type: ignore message=error_str, model=model, @@ -429,7 +473,10 @@ def completion_cost( prompt_characters = 0 completion_tokens = 0 completion_characters = 0 - if completion_response is not None: + if completion_response is not None and ( + isinstance(completion_response, BaseModel) + or isinstance(completion_response, dict) + ): # tts returns a custom class # get input/output tokens from completion_response prompt_tokens = completion_response.get("usage", {}).get("prompt_tokens", 0) completion_tokens = completion_response.get("usage", {}).get( @@ -535,6 +582,11 @@ def completion_cost( raise Exception( f"Model={image_gen_model_name} not found in completion cost model map" ) + elif ( + call_type == CallTypes.speech.value or call_type == CallTypes.aspeech.value + ): + prompt_characters = litellm.utils._count_characters(text=prompt) + # Calculate cost based on prompt_tokens, completion_tokens if ( "togethercomputer" in model @@ -591,6 +643,7 @@ def completion_cost( custom_cost_per_token=custom_cost_per_token, prompt_characters=prompt_characters, completion_characters=completion_characters, + call_type=call_type, ) _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar print_verbose( @@ -608,6 +661,7 @@ def response_cost_calculator( ImageResponse, TranscriptionResponse, TextCompletionResponse, + HttpxBinaryResponseContent, ], model: str, custom_llm_provider: Optional[str], @@ -641,7 +695,8 @@ def response_cost_calculator( if cache_hit is not None and cache_hit is True: response_cost = 0.0 else: - response_object._hidden_params["optional_params"] = optional_params + if isinstance(response_object, BaseModel): + response_object._hidden_params["optional_params"] = optional_params if isinstance(response_object, ImageResponse): response_cost = completion_cost( completion_response=response_object, @@ -651,12 +706,11 @@ def response_cost_calculator( ) else: if ( - model in litellm.model_cost - and custom_pricing is not None - and custom_llm_provider is True + model in litellm.model_cost or custom_pricing is True ): # override defaults if custom pricing is set base_model = model # base_model defaults to None if not set on model_info + response_cost = completion_cost( completion_response=response_object, call_type=call_type, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 381bcc1ac94..f9f32552da0 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -24,6 +24,8 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, ) +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS from litellm.types.utils import ( CallTypes, EmbeddingResponse, @@ -517,33 +519,36 @@ class Logging: self.model_call_details["cache_hit"] = cache_hit ## if model in model cost map - log the response cost ## else set cost to None - verbose_logger.debug(f"Model={self.model};") if ( - result is not None - and ( + result is not None and self.stream is not True + ): # handle streaming separately + if ( isinstance(result, ModelResponse) or isinstance(result, EmbeddingResponse) or isinstance(result, ImageResponse) or isinstance(result, TranscriptionResponse) or isinstance(result, TextCompletionResponse) - ) - and self.stream != True - ): # handle streaming separately - self.model_call_details["response_cost"] = ( - litellm.response_cost_calculator( - response_object=result, - model=self.model, - cache_hit=self.model_call_details.get("cache_hit", False), - custom_llm_provider=self.model_call_details.get( - "custom_llm_provider", None - ), - base_model=_get_base_model_from_metadata( - model_call_details=self.model_call_details - ), - call_type=self.call_type, - optional_params=self.optional_params, + or isinstance(result, HttpxBinaryResponseContent) # tts + ): + custom_pricing = use_custom_pricing_for_model( + litellm_params=self.litellm_params + ) + self.model_call_details["response_cost"] = ( + litellm.response_cost_calculator( + response_object=result, + model=self.model, + cache_hit=self.model_call_details.get("cache_hit", False), + custom_llm_provider=self.model_call_details.get( + "custom_llm_provider", None + ), + base_model=_get_base_model_from_metadata( + model_call_details=self.model_call_details + ), + call_type=self.call_type, + optional_params=self.optional_params, + custom_pricing=custom_pricing, + ) ) - ) else: # streaming chunks + image gen. self.model_call_details["response_cost"] = None @@ -2012,3 +2017,17 @@ def get_custom_logger_compatible_class( if isinstance(callback, _PROXY_DynamicRateLimitHandler): return callback # type: ignore return None + + +def use_custom_pricing_for_model(litellm_params: Optional[dict]) -> bool: + if litellm_params is None: + return False + metadata: Optional[dict] = litellm_params.get("metadata", {}) + if metadata is None: + return False + model_info: Optional[dict] = metadata.get("model_info", {}) + if model_info is not None: + for k, v in model_info.items(): + if k in SPECIAL_MODEL_INFO_PARAMS: + return True + return False diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py new file mode 100644 index 00000000000..e986a22a6c9 --- /dev/null +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -0,0 +1,85 @@ +# What is this? +## Helper utilities for cost_per_token() + +import traceback +from typing import List, Literal, Optional, Tuple + +import litellm +from litellm import verbose_logger + + +def _generic_cost_per_character( + model: str, + custom_llm_provider: str, + prompt_characters: float, + completion_characters: float, + custom_prompt_cost: Optional[float], + custom_completion_cost: Optional[float], +) -> Tuple[Optional[float], Optional[float]]: + """ + Generic function to help calculate cost per character. + """ + """ + Calculates the cost per character for a given model, input messages, and response object. + + Input: + - model: str, the model name without provider prefix + - custom_llm_provider: str, "vertex_ai-*" + - prompt_characters: float, the number of input characters + - completion_characters: float, the number of output characters + + Returns: + Tuple[Optional[float], Optional[float]] - prompt_cost_in_usd, completion_cost_in_usd. + - returns None if not able to calculate cost. + + Raises: + Exception if 'input_cost_per_character' or 'output_cost_per_character' is missing from model_info + """ + args = locals() + ## GET MODEL INFO + model_info = litellm.get_model_info( + model=model, custom_llm_provider=custom_llm_provider + ) + + ## CALCULATE INPUT COST + try: + if custom_prompt_cost is None: + assert ( + "input_cost_per_character" in model_info + and model_info["input_cost_per_character"] is not None + ), "model info for model={} does not have 'input_cost_per_character'-pricing\nmodel_info={}".format( + model, model_info + ) + custom_prompt_cost = model_info["input_cost_per_character"] + + prompt_cost = prompt_characters * custom_prompt_cost + except Exception as e: + verbose_logger.error( + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\n{}\nDefaulting to None".format( + str(e), traceback.format_exc() + ) + ) + + prompt_cost = None + + ## CALCULATE OUTPUT COST + try: + if custom_completion_cost is None: + assert ( + "output_cost_per_character" in model_info + and model_info["output_cost_per_character"] is not None + ), "model info for model={} does not have 'output_cost_per_character'-pricing\nmodel_info={}".format( + model, model_info + ) + custom_completion_cost = model_info["output_cost_per_character"] + completion_cost = completion_characters * custom_completion_cost + except Exception as e: + verbose_logger.error( + "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\n{}\nDefaulting to None".format( + str(e), traceback.format_exc() + ) + ) + + completion_cost = None + + return prompt_cost, completion_cost diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index a7a5c8bf1a2..a7d46d5061e 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,5 +1,5 @@ model_list: - - model_name: "*" + - model_name: tts litellm_params: model: "openai/*" litellm_settings: diff --git a/litellm/tests/test_completion_cost.py b/litellm/tests/test_completion_cost.py index bffb68e0e5d..1b4df0ecc08 100644 --- a/litellm/tests/test_completion_cost.py +++ b/litellm/tests/test_completion_cost.py @@ -712,7 +712,6 @@ def test_vertex_ai_claude_completion_cost(): assert cost == predicted_cost - @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_completion_cost_hidden_params(sync_mode): @@ -732,6 +731,7 @@ async def test_completion_cost_hidden_params(sync_mode): assert "response_cost" in response._hidden_params assert isinstance(response._hidden_params["response_cost"], float) + def test_vertex_ai_gemini_predict_cost(): model = "gemini-1.5-flash" messages = [{"role": "user", "content": "Hey, hows it going???"}] @@ -739,3 +739,16 @@ def test_vertex_ai_gemini_predict_cost(): assert predictive_cost > 0 + +@pytest.mark.parametrize("model", ["openai/tts-1", "azure/tts-1"]) +def test_completion_cost_tts(model): + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + cost = completion_cost( + model=model, + prompt="the quick brown fox jumped over the lazy dogs", + call_type="speech", + ) + + assert cost > 0 diff --git a/litellm/types/router.py b/litellm/types/router.py index 46fc0c9e7eb..9c028fa87a0 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -324,7 +324,12 @@ class DeploymentTypedDict(TypedDict): litellm_params: LiteLLMParamsTypedDict -SPECIAL_MODEL_INFO_PARAMS = ["input_cost_per_token", "output_cost_per_token"] +SPECIAL_MODEL_INFO_PARAMS = [ + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_character", + "output_cost_per_character", +] class Deployment(BaseModel): diff --git a/litellm/utils.py b/litellm/utils.py index a28531b6c13..d17ba8911c0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4707,7 +4707,9 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod ) except Exception: raise Exception( - "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json" + "This model isn't mapped yet. model={}, custom_llm_provider={}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json".format( + model, custom_llm_provider + ) )