diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a3cb847a4f6..4b5ac51db97 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -410,6 +410,35 @@ def get_replicate_completion_pricing(completion_response=None, total_time=0.0): return a100_80gb_price_per_second_public * total_time / 1000 +def _select_model_name_for_cost_calc( + model: Optional[str], + completion_response: Union[BaseModel, dict], + base_model: Optional[str] = None, + custom_pricing: Optional[bool] = None, +) -> Optional[str]: + """ + 1. If custom pricing is true, return received model name + 2. If base_model is set (e.g. for azure models), return that + 3. If completion response has model set return that + 4. If model is passed in return that + """ + if custom_pricing is True: + return model + + if base_model is not None: + return base_model + + return_model = model or completion_response.get("model", "") # type: ignore + if hasattr(completion_response, "_hidden_params"): + if ( + completion_response._hidden_params.get("model", None) is not None + and len(completion_response._hidden_params["model"]) > 0 + ): + return_model = completion_response._hidden_params.get("model", model) + + return return_model + + def completion_cost( completion_response=None, model: Optional[str] = None, @@ -511,15 +540,10 @@ def completion_cost( verbose_logger.debug( f"completion_response response ms: {getattr(completion_response, '_response_ms', None)} " ) - model = model or completion_response.get( - "model", None - ) # check if user passed an override for model, if it's none check completion_response['model'] + model = _select_model_name_for_cost_calc( + model=model, completion_response=completion_response + ) if hasattr(completion_response, "_hidden_params"): - if ( - completion_response._hidden_params.get("model", None) is not None - and len(completion_response._hidden_params["model"]) > 0 - ): - model = completion_response._hidden_params.get("model", model) custom_llm_provider = completion_response._hidden_params.get( "custom_llm_provider", custom_llm_provider or "" ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c389a1737c1..74c6d0db018 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -24,6 +24,7 @@ from litellm import ( verbose_logger, ) from litellm.caching import DualCache, InMemoryCache, S3Cache +from litellm.cost_calculator import _select_model_name_for_cost_calc from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.redact_messages import ( redact_message_input_output_from_logging, @@ -37,6 +38,7 @@ from litellm.types.utils import ( ModelResponse, StandardLoggingHiddenParams, StandardLoggingMetadata, + StandardLoggingModelInformation, StandardLoggingPayload, TextCompletionResponse, TranscriptionResponse, @@ -2293,6 +2295,38 @@ def get_standard_logging_object_payload( id = f"{id}_cache_hit{time.time()}" # do not duplicate the request id + ## Get model cost information ## + base_model = _get_base_model_from_metadata(model_call_details=kwargs) + custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params) + model_cost_name = _select_model_name_for_cost_calc( + model=None, + completion_response=init_response_obj, + base_model=base_model, + custom_pricing=custom_pricing, + ) + if model_cost_name is None: + model_cost_information = StandardLoggingModelInformation( + model_map_key="", model_map_value=None + ) + else: + custom_llm_provider = kwargs.get("custom_llm_provider", None) + + try: + _model_cost_information = litellm.get_model_info( + model=model_cost_name, custom_llm_provider=custom_llm_provider + ) + model_cost_information = StandardLoggingModelInformation( + model_map_key=model_cost_name, + model_map_value=_model_cost_information, + ) + except Exception: + verbose_logger.warning( + "Model is not mapped in model cost map. Defaulting to None model_cost_information for standard_logging_payload" + ) + model_cost_information = StandardLoggingModelInformation( + model_map_key=model_cost_name, model_map_value=None + ) + payload: StandardLoggingPayload = StandardLoggingPayload( id=str(id), call_type=call_type or "", @@ -2319,6 +2353,7 @@ def get_standard_logging_object_payload( ), model_parameters=kwargs.get("optional_params", None), hidden_params=clean_hidden_params, + model_map_information=model_cost_information, ) verbose_logger.debug( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index fcbc4a17005..9c35db02f19 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -3,4 +3,4 @@ model_list: litellm_params: model: "azure/azure-embedding-model" api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY \ No newline at end of file + api_key: os.environ/AZURE_API_KEY diff --git a/litellm/tests/test_completion_cost.py b/litellm/tests/test_completion_cost.py index 8df4791c71b..9c5075c081d 100644 --- a/litellm/tests/test_completion_cost.py +++ b/litellm/tests/test_completion_cost.py @@ -251,7 +251,7 @@ def test_cost_azure_gpt_35(): ) cost = litellm.completion_cost( - completion_response=resp, model="azure/gpt-35-turbo" + completion_response=resp, model="azure/chatgpt-v-2" ) print("\n Calculated Cost for azure/gpt-3.5-turbo", cost) input_cost = model_cost["azure/gpt-35-turbo"]["input_cost_per_token"] @@ -262,9 +262,7 @@ def test_cost_azure_gpt_35(): print("\n Excpected cost", expected_cost) assert cost == expected_cost except Exception as e: - pytest.fail( - f"Cost Calc failed for azure/gpt-3.5-turbo. Expected {expected_cost}, Calculated cost {cost}" - ) + pytest.fail(f"Cost Calc failed for azure/gpt-3.5-turbo. {str(e)}") # test_cost_azure_gpt_35() diff --git a/litellm/tests/test_custom_callback_input.py b/litellm/tests/test_custom_callback_input.py index a356ee0adce..ffec5ac7dcf 100644 --- a/litellm/tests/test_custom_callback_input.py +++ b/litellm/tests/test_custom_callback_input.py @@ -1171,7 +1171,8 @@ def test_turn_off_message_logging(): ##### VALID JSON ###### -def test_standard_logging_payload(): +@pytest.mark.parametrize("model", ["gpt-3.5-turbo", "azure/chatgpt-v-2"]) +def test_standard_logging_payload(model): """ Ensure valid standard_logging_payload is passed for logging calls to s3 @@ -1187,9 +1188,9 @@ def test_standard_logging_payload(): customHandler, "log_success_event", new=MagicMock() ) as mock_client: _ = litellm.completion( - model="gpt-3.5-turbo", + model=model, messages=[{"role": "user", "content": "Hey, how's it going?"}], - mock_response="Going well!", + # mock_response="Going well!", ) time.sleep(2) @@ -1204,7 +1205,11 @@ def test_standard_logging_payload(): is not None ) - print(mock_client.call_args.kwargs["kwargs"]["standard_logging_object"]) + print( + "Standard Logging Object - {}".format( + mock_client.call_args.kwargs["kwargs"]["standard_logging_object"] + ) + ) keys_list = list(StandardLoggingPayload.__annotations__.keys()) @@ -1226,3 +1231,9 @@ def test_standard_logging_payload(): ] > 0 ) + assert ( + mock_client.call_args.kwargs["kwargs"]["standard_logging_object"][ + "model_map_information" + ]["model_map_value"] + is not None + ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c78cc2edcf8..a65701d2fd7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1195,6 +1195,11 @@ class StandardLoggingHiddenParams(TypedDict): additional_headers: Optional[dict] +class StandardLoggingModelInformation(TypedDict): + model_map_key: str + model_map_value: Optional[ModelInfo] + + class StandardLoggingPayload(TypedDict): id: str call_type: str @@ -1205,6 +1210,7 @@ class StandardLoggingPayload(TypedDict): startTime: float endTime: float completionStartTime: float + model_map_information: StandardLoggingModelInformation model: str model_id: Optional[str] model_group: Optional[str]