Merge pull request #4572 from BerriAI/litellm_proxy_tts_pricing

Azure proxy tts pricing
2026-10-07 02:59:05 +00:00 · 2024-07-06 14:56:28 -07:00 · 2024-07-06 14:56:28 -07:00 · 128c752f89
commit 128c752f89
parent 27e9f96380 b3ef1ac18c
7 changed files with 208 additions and 30 deletions
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@ -4,6 +4,8 @@ import time
 import traceback
 from typing import List, Literal, Optional, Tuple, Union

+from pydantic import BaseModel
+
 import litellm
 import litellm._logging
 from litellm import verbose_logger
@ -13,6 +15,9 @@ from litellm.litellm_core_utils.llm_cost_calc.google import (
 from litellm.litellm_core_utils.llm_cost_calc.google import (
    cost_per_token as google_cost_per_token,
 )
+from litellm.litellm_core_utils.llm_cost_calc.utils import _generic_cost_per_character
+from litellm.types.llms.openai import HttpxBinaryResponseContent
+from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS
 from litellm.utils import (
    CallTypes,
    CostPerToken,
@ -62,6 +67,23 @@ def cost_per_token(
    ### CUSTOM PRICING ###
    custom_cost_per_token: Optional[CostPerToken] = None,
    custom_cost_per_second: Optional[float] = None,
+    ### CALL TYPE ###
+    call_type: Literal[
+        "embedding",
+        "aembedding",
+        "completion",
+        "acompletion",
+        "atext_completion",
+        "text_completion",
+        "image_generation",
+        "aimage_generation",
+        "moderation",
+        "amoderation",
+        "atranscription",
+        "transcription",
+        "aspeech",
+        "speech",
+    ] = "completion",
 ) -> Tuple[float, float]:
    """
    Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@ -76,6 +98,7 @@ def cost_per_token(
        custom_llm_provider (str): The llm provider to whom the call was made (see init.py for full list)
        custom_cost_per_token: Optional[CostPerToken]: the cost per input + output token for the llm api call.
        custom_cost_per_second: Optional[float]: the cost per second for the llm api call.
+        call_type: Optional[str]: the call type

    Returns:
        tuple: A tuple containing the cost in USD dollars for prompt tokens and completion tokens, respectively.
@ -159,6 +182,27 @@ def cost_per_token(
            prompt_tokens=prompt_tokens,
            completion_tokens=completion_tokens,
        )
+    elif call_type == "speech" or call_type == "aspeech":
+        prompt_cost, completion_cost = _generic_cost_per_character(
+            model=model_without_prefix,
+            custom_llm_provider=custom_llm_provider,
+            prompt_characters=prompt_characters,
+            completion_characters=completion_characters,
+            custom_prompt_cost=None,
+            custom_completion_cost=0,
+        )
+        if prompt_cost is None or completion_cost is None:
+            raise ValueError(
+                "cost for tts call is None. prompt_cost={}, completion_cost={}, model={}, custom_llm_provider={}, prompt_characters={}, completion_characters={}".format(
+                    prompt_cost,
+                    completion_cost,
+                    model_without_prefix,
+                    custom_llm_provider,
+                    prompt_characters,
+                    completion_characters,
+                )
+            )
+        return prompt_cost, completion_cost
    elif model in model_cost_ref:
        print_verbose(f"Success: model={model} in model_cost_map")
        print_verbose(
@ -289,7 +333,7 @@ def cost_per_token(
        return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
    else:
        # if model is not in model_prices_and_context_window.json. Raise an exception-let users know
-        error_str = f"Model not in model_prices_and_context_window.json. You passed model={model}. Register pricing for model - https://docs.litellm.ai/docs/proxy/custom_pricing\n"
+        error_str = f"Model not in model_prices_and_context_window.json. You passed model={model}, custom_llm_provider={custom_llm_provider}. Register pricing for model - https://docs.litellm.ai/docs/proxy/custom_pricing\n"
        raise litellm.exceptions.NotFoundError(  # type: ignore
            message=error_str,
            model=model,
@ -429,7 +473,10 @@ def completion_cost(
        prompt_characters = 0
        completion_tokens = 0
        completion_characters = 0
-        if completion_response is not None:
+        if completion_response is not None and (
+            isinstance(completion_response, BaseModel)
+            or isinstance(completion_response, dict)
+        ):  # tts returns a custom class
            # get input/output tokens from completion_response
            prompt_tokens = completion_response.get("usage", {}).get("prompt_tokens", 0)
            completion_tokens = completion_response.get("usage", {}).get(
@ -535,6 +582,11 @@ def completion_cost(
                raise Exception(
                    f"Model={image_gen_model_name} not found in completion cost model map"
                )
+        elif (
+            call_type == CallTypes.speech.value or call_type == CallTypes.aspeech.value
+        ):
+            prompt_characters = litellm.utils._count_characters(text=prompt)
+
        # Calculate cost based on prompt_tokens, completion_tokens
        if (
            "togethercomputer" in model
@ -591,6 +643,7 @@ def completion_cost(
            custom_cost_per_token=custom_cost_per_token,
            prompt_characters=prompt_characters,
            completion_characters=completion_characters,
+            call_type=call_type,
        )
        _final_cost = prompt_tokens_cost_usd_dollar + completion_tokens_cost_usd_dollar
        print_verbose(
@ -608,6 +661,7 @@ def response_cost_calculator(
        ImageResponse,
        TranscriptionResponse,
        TextCompletionResponse,
+        HttpxBinaryResponseContent,
    ],
    model: str,
    custom_llm_provider: Optional[str],
@ -641,7 +695,8 @@ def response_cost_calculator(
        if cache_hit is not None and cache_hit is True:
            response_cost = 0.0
        else:
-            response_object._hidden_params["optional_params"] = optional_params
+            if isinstance(response_object, BaseModel):
+                response_object._hidden_params["optional_params"] = optional_params
            if isinstance(response_object, ImageResponse):
                response_cost = completion_cost(
                    completion_response=response_object,
@ -651,12 +706,11 @@ def response_cost_calculator(
                )
            else:
                if (
-                    model in litellm.model_cost
-                    and custom_pricing is not None
-                    and custom_llm_provider is True
+                    model in litellm.model_cost or custom_pricing is True
                ):  # override defaults if custom pricing is set
                    base_model = model
                # base_model defaults to None if not set on model_info
+
                response_cost = completion_cost(
                    completion_response=response_object,
                    call_type=call_type,
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@ -24,6 +24,8 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.redact_messages import (
    redact_message_input_output_from_logging,
 )
+from litellm.types.llms.openai import HttpxBinaryResponseContent
+from litellm.types.router import SPECIAL_MODEL_INFO_PARAMS
 from litellm.types.utils import (
    CallTypes,
    EmbeddingResponse,
@ -517,33 +519,36 @@ class Logging:
            self.model_call_details["cache_hit"] = cache_hit
            ## if model in model cost map - log the response cost
            ## else set cost to None
-            verbose_logger.debug(f"Model={self.model};")
            if (
-                result is not None
-                and (
+                result is not None and self.stream is not True
+            ):  # handle streaming separately
+                if (
                    isinstance(result, ModelResponse)
                    or isinstance(result, EmbeddingResponse)
                    or isinstance(result, ImageResponse)
                    or isinstance(result, TranscriptionResponse)
                    or isinstance(result, TextCompletionResponse)
-                )
-                and self.stream != True
-            ):  # handle streaming separately
-                self.model_call_details["response_cost"] = (
-                    litellm.response_cost_calculator(
-                        response_object=result,
-                        model=self.model,
-                        cache_hit=self.model_call_details.get("cache_hit", False),
-                        custom_llm_provider=self.model_call_details.get(
-                            "custom_llm_provider", None
-                        ),
-                        base_model=_get_base_model_from_metadata(
-                            model_call_details=self.model_call_details
-                        ),
-                        call_type=self.call_type,
-                        optional_params=self.optional_params,
+                    or isinstance(result, HttpxBinaryResponseContent)  # tts
+                ):
+                    custom_pricing = use_custom_pricing_for_model(
+                        litellm_params=self.litellm_params
+                    )
+                    self.model_call_details["response_cost"] = (
+                        litellm.response_cost_calculator(
+                            response_object=result,
+                            model=self.model,
+                            cache_hit=self.model_call_details.get("cache_hit", False),
+                            custom_llm_provider=self.model_call_details.get(
+                                "custom_llm_provider", None
+                            ),
+                            base_model=_get_base_model_from_metadata(
+                                model_call_details=self.model_call_details
+                            ),
+                            call_type=self.call_type,
+                            optional_params=self.optional_params,
+                            custom_pricing=custom_pricing,
+                        )
                    )
-                )
            else:  # streaming chunks + image gen.
                self.model_call_details["response_cost"] = None

@ -2012,3 +2017,17 @@ def get_custom_logger_compatible_class(
            if isinstance(callback, _PROXY_DynamicRateLimitHandler):
                return callback  # type: ignore
    return None
+
+
+def use_custom_pricing_for_model(litellm_params: Optional[dict]) -> bool:
+    if litellm_params is None:
+        return False
+    metadata: Optional[dict] = litellm_params.get("metadata", {})
+    if metadata is None:
+        return False
+    model_info: Optional[dict] = metadata.get("model_info", {})
+    if model_info is not None:
+        for k, v in model_info.items():
+            if k in SPECIAL_MODEL_INFO_PARAMS:
+                return True
+    return False
--- a/litellm/litellm_core_utils/llm_cost_calc/utils.py
+++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py
@ -0,0 +1,85 @@
+# What is this?
+## Helper utilities for cost_per_token()
+
+import traceback
+from typing import List, Literal, Optional, Tuple
+
+import litellm
+from litellm import verbose_logger
+
+
+def _generic_cost_per_character(
+    model: str,
+    custom_llm_provider: str,
+    prompt_characters: float,
+    completion_characters: float,
+    custom_prompt_cost: Optional[float],
+    custom_completion_cost: Optional[float],
+) -> Tuple[Optional[float], Optional[float]]:
+    """
+    Generic function to help calculate cost per character.
+    """
+    """
+    Calculates the cost per character for a given model, input messages, and response object.
+
+    Input:
+        - model: str, the model name without provider prefix
+        - custom_llm_provider: str, "vertex_ai-*"
+        - prompt_characters: float, the number of input characters
+        - completion_characters: float, the number of output characters
+
+    Returns:
+        Tuple[Optional[float], Optional[float]] - prompt_cost_in_usd, completion_cost_in_usd. 
+        - returns None if not able to calculate cost.
+
+    Raises:
+        Exception if 'input_cost_per_character' or 'output_cost_per_character' is missing from model_info
+    """
+    args = locals()
+    ## GET MODEL INFO
+    model_info = litellm.get_model_info(
+        model=model, custom_llm_provider=custom_llm_provider
+    )
+
+    ## CALCULATE INPUT COST
+    try:
+        if custom_prompt_cost is None:
+            assert (
+                "input_cost_per_character" in model_info
+                and model_info["input_cost_per_character"] is not None
+            ), "model info for model={} does not have 'input_cost_per_character'-pricing\nmodel_info={}".format(
+                model, model_info
+            )
+            custom_prompt_cost = model_info["input_cost_per_character"]
+
+        prompt_cost = prompt_characters * custom_prompt_cost
+    except Exception as e:
+        verbose_logger.error(
+            "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\n{}\nDefaulting to None".format(
+                str(e), traceback.format_exc()
+            )
+        )
+
+        prompt_cost = None
+
+    ## CALCULATE OUTPUT COST
+    try:
+        if custom_completion_cost is None:
+            assert (
+                "output_cost_per_character" in model_info
+                and model_info["output_cost_per_character"] is not None
+            ), "model info for model={} does not have 'output_cost_per_character'-pricing\nmodel_info={}".format(
+                model, model_info
+            )
+            custom_completion_cost = model_info["output_cost_per_character"]
+        completion_cost = completion_characters * custom_completion_cost
+    except Exception as e:
+        verbose_logger.error(
+            "litellm.litellm_core_utils.llm_cost_calc.utils.py::cost_per_character(): Exception occured - {}\n{}\nDefaulting to None".format(
+                str(e), traceback.format_exc()
+            )
+        )
+
+        completion_cost = None
+
+    return prompt_cost, completion_cost
--- a/litellm/proxy/_new_secret_config.yaml
+++ b/litellm/proxy/_new_secret_config.yaml
@ -1,5 +1,5 @@
 model_list:
-  - model_name: "*"             
+  - model_name: tts
    litellm_params:
      model: "openai/*"
 litellm_settings:
--- a/litellm/tests/test_completion_cost.py
+++ b/litellm/tests/test_completion_cost.py
@ -712,7 +712,6 @@ def test_vertex_ai_claude_completion_cost():
    assert cost == predicted_cost


-
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.asyncio
 async def test_completion_cost_hidden_params(sync_mode):
@ -732,6 +731,7 @@ async def test_completion_cost_hidden_params(sync_mode):
    assert "response_cost" in response._hidden_params
    assert isinstance(response._hidden_params["response_cost"], float)

+
 def test_vertex_ai_gemini_predict_cost():
    model = "gemini-1.5-flash"
    messages = [{"role": "user", "content": "Hey, hows it going???"}]
@ -739,3 +739,16 @@ def test_vertex_ai_gemini_predict_cost():

    assert predictive_cost > 0

+
+@pytest.mark.parametrize("model", ["openai/tts-1", "azure/tts-1"])
+def test_completion_cost_tts(model):
+    os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+
+    cost = completion_cost(
+        model=model,
+        prompt="the quick brown fox jumped over the lazy dogs",
+        call_type="speech",
+    )
+
+    assert cost > 0
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@ -324,7 +324,12 @@ class DeploymentTypedDict(TypedDict):
    litellm_params: LiteLLMParamsTypedDict


-SPECIAL_MODEL_INFO_PARAMS = ["input_cost_per_token", "output_cost_per_token"]
+SPECIAL_MODEL_INFO_PARAMS = [
+    "input_cost_per_token",
+    "output_cost_per_token",
+    "input_cost_per_character",
+    "output_cost_per_character",
+]


 class Deployment(BaseModel):
--- a/litellm/utils.py
+++ b/litellm/utils.py
@ -4707,7 +4707,9 @@ def get_model_info(model: str, custom_llm_provider: Optional[str] = None) -> Mod
            )
    except Exception:
        raise Exception(
-            "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
+            "This model isn't mapped yet. model={}, custom_llm_provider={}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json".format(
+                model, custom_llm_provider
+            )
        )