diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index d5d95639aa7..d22474a9dd8 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5251,18 +5251,26 @@ def get_standard_logging_object_payload( # For anthropic_messages call type, follow Anthropic convention: # prompt_tokens should NOT include cached tokens if call_type == CallTypes.anthropic_messages.value and usage_dict: - prompt_tokens_details = usage_dict.get("prompt_tokens_details", {}) - if isinstance(prompt_tokens_details, dict): - cached_tokens = prompt_tokens_details.get("cached_tokens", 0) or 0 - cache_creation_tokens = ( - prompt_tokens_details.get("cache_creation_tokens", 0) or 0 - ) - if cached_tokens > 0 or cache_creation_tokens > 0: - current_prompt_tokens = usage_dict.get("prompt_tokens", 0) or 0 - adjusted_prompt_tokens = ( - current_prompt_tokens - cached_tokens - cache_creation_tokens + import copy + + usage_dict = copy.deepcopy(usage_dict) + prompt_tokens_details = usage_dict.get("prompt_tokens_details") + if prompt_tokens_details is not None: + if isinstance(prompt_tokens_details, BaseModel): + prompt_tokens_details = prompt_tokens_details.model_dump() + if isinstance(prompt_tokens_details, dict): + cached_tokens = prompt_tokens_details.get("cached_tokens", 0) or 0 + cache_creation_tokens = ( + prompt_tokens_details.get("cache_creation_tokens", 0) or 0 ) - usage_dict["prompt_tokens"] = max(0, adjusted_prompt_tokens) + if cached_tokens > 0 or cache_creation_tokens > 0: + current_prompt_tokens = usage_dict.get("prompt_tokens", 0) or 0 + adjusted_prompt_tokens = ( + current_prompt_tokens + - cached_tokens + - cache_creation_tokens + ) + usage_dict["prompt_tokens"] = max(0, adjusted_prompt_tokens) id = response_obj.get("id", kwargs.get("litellm_call_id")) diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 1155407c193..8d9275682a7 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -430,6 +430,17 @@ def get_logging_payload( # noqa: PLR0915 model_name = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {}) try: + # For anthropic_messages, prioritize standard_logging_payload tokens + # (which exclude cached tokens per Anthropic convention) + if call_type == "anthropic_messages" and standard_logging_prompt_tokens is not None: + db_prompt_tokens = standard_logging_prompt_tokens + db_completion_tokens = standard_logging_completion_tokens or 0 + db_total_tokens = standard_logging_total_tokens or 0 + else: + db_prompt_tokens = usage.get("prompt_tokens", standard_logging_prompt_tokens or 0) + db_completion_tokens = usage.get("completion_tokens", standard_logging_completion_tokens or 0) + db_total_tokens = usage.get("total_tokens", standard_logging_total_tokens or 0) + payload: SpendLogsPayload = SpendLogsPayload( request_id=str(id), call_type=call_type or "", @@ -445,11 +456,9 @@ def get_logging_payload( # noqa: PLR0915 metadata=safe_dumps(clean_metadata), cache_key=cache_key, spend=kwargs.get("response_cost", 0), - total_tokens=standard_logging_total_tokens if standard_logging_total_tokens is not None else usage.get("total_tokens", 0), - prompt_tokens=standard_logging_prompt_tokens if standard_logging_prompt_tokens is not None else usage.get("prompt_tokens", 0), - completion_tokens=standard_logging_completion_tokens if standard_logging_completion_tokens is not None else usage.get( - "completion_tokens", 0 - ), + total_tokens=db_total_tokens, + prompt_tokens=db_prompt_tokens, + completion_tokens=db_completion_tokens, request_tags=request_tags, end_user=end_user_id or "", api_base=litellm_params.get("api_base", ""), diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 8ed728c5b28..6d5b4d1f756 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -1,231 +1,21 @@ model_list: - - model_name: gpt-3.5-turbo-end-user-test + - model_name: anthropic/* litellm_params: - model: gpt-3.5-turbo - region_name: "eu" - model_info: - id: "1" - - model_name: gpt-3.5-turbo-end-user-test + model: anthropic/* + - model_name: azure_ai/claude-opus-4-6 litellm_params: - model: openai/gpt-4.1-mini - api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - - model_name: gpt-3.5-turbo - litellm_params: - model: openai/gpt-4.1-mini - api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - - model_name: gpt-3.5-turbo-large - litellm_params: - model: "gpt-3.5-turbo-1106" - api_key: os.environ/OPENAI_API_KEY - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: gpt-4 - litellm_params: - model: openai/gpt-4.1-mini - api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault - rpm: 480 - timeout: 300 - stream_timeout: 60 - - model_name: sagemaker-completion-model - litellm_params: - model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4 - input_cost_per_second: 0.000420 - - model_name: text-embedding-ada-002 - litellm_params: - model: openai/text-embedding-ada-002 - api_key: os.environ/OPENAI_API_KEY - model_info: - mode: embedding - base_model: text-embedding-ada-002 - - model_name: dall-e-2 # some tests use dall-e-2 which is now deprecated, alias to dall-e-3 - litellm_params: - model: openai/dall-e-3 - - model_name: openai-dall-e-3 - litellm_params: - model: dall-e-3 - - model_name: fake-openai-endpoint - litellm_params: - model: openai/gpt-3.5-turbo-0301 - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - - model_name: fake-openai-endpoint-2 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - stream_timeout: 0.001 - rpm: 1 - - model_name: fake-openai-endpoint-3 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - stream_timeout: 0.001 - rpm: 1000 - - model_name: fake-openai-endpoint-4 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - num_retries: 50 - - model_name: fake-openai-endpoint-3 - litellm_params: - model: openai/my-fake-model-2 - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - stream_timeout: 0.001 - rpm: 1000 - - model_name: bad-model - litellm_params: - model: openai/bad-model - api_key: os.environ/OPENAI_API_KEY - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - mock_timeout: True - timeout: 60 - rpm: 1000 - model_info: - health_check_timeout: 1 - - model_name: good-model - litellm_params: - model: openai/bad-model - api_key: os.environ/OPENAI_API_KEY - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - rpm: 1000 - model_info: - health_check_timeout: 1 - - model_name: "*" - litellm_params: - model: openai/* - api_key: os.environ/OPENAI_API_KEY - - model_name: realtime-v1 - litellm_params: - model: azure/gpt-realtime-20250828-standard - api_version: "2025-08-28" - realtime_protocol: GA # Possible values: "GA"/ "v1", "beta" + model: azure_ai/claude-opus-4-6 + api_base: https://ai-eisgptphiai810714567559.services.ai.azure.com + api_key: os.environ/AZURE_AI_API_KEY + cache_control_injection_points: + - role: system + location: message + drop_params: true - - model_name: realtime-beta - litellm_params: - model: azure/gpt-realtime-20250828-standard - api_version: 2025-04-01-preview - - - # provider specific wildcard routing - - model_name: "anthropic/*" - litellm_params: - model: "anthropic/*" - api_key: os.environ/ANTHROPIC_API_KEY - - model_name: "bedrock/*" - litellm_params: - model: "bedrock/*" - - model_name: "groq/*" - litellm_params: - model: "groq/*" - api_key: os.environ/GROQ_API_KEY - - model_name: mistral-embed - litellm_params: - model: mistral/mistral-embed - - model_name: gpt-instruct # [PROD TEST] - tests if `/health` automatically infers this to be a text completion model - litellm_params: - model: text-completion-openai/gpt-3.5-turbo-instruct - - model_name: fake-openai-endpoint-5 - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - timeout: 1 - - model_name: badly-configured-openai-endpoint - litellm_params: - model: openai/my-fake-model - api_key: my-fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.appxxxx/ - - model_name: gemini-1.5-flash - litellm_params: - model: gemini/gemini-1.5-flash - api_key: os.environ/GOOGLE_API_KEY - - model_name: gpt-4o - litellm_params: - model: gpt-4o - api_key: os.environ/OPENAI_API_KEY - - -litellm_settings: - # set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production - drop_params: True - success_callback: ["prometheus"] - # max_budget: 100 - # budget_duration: 30d - num_retries: 5 - request_timeout: 600 - telemetry: False - context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}] - default_team_settings: - - team_id: team-1 - success_callback: ["langfuse"] - failure_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1 - langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1 - - team_id: team-2 - success_callback: ["langfuse"] - failure_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2 - langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2 - langfuse_host: https://us.cloud.langfuse.com - # cache: true # [OPTIONAL] use for caching responses - # enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys - # cache_params: # And for shared health check - # type: redis - # host: localhost - # port: 6379 - -# For /fine_tuning/jobs endpoints -finetune_settings: - - custom_llm_provider: azure - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2023-03-15-preview" - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY - -# for /files endpoints -files_settings: - - custom_llm_provider: azure - api_base: os.environ/AZURE_API_BASE - api_key: os.environ/AZURE_API_KEY - api_version: "2023-03-15-preview" - - custom_llm_provider: openai - api_key: os.environ/OPENAI_API_KEY router_settings: - routing_strategy: usage-based-routing-v2 - redis_host: os.environ/REDIS_HOST - redis_password: os.environ/REDIS_PASSWORD - redis_port: os.environ/REDIS_PORT + routing_strategy: usage-based-routing-v2 enable_pre_call_checks: true - model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"} -general_settings: - master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys - store_model_in_db: True - proxy_budget_rescheduler_min_time: 60 - proxy_budget_rescheduler_max_time: 64 - proxy_batch_write_at: 1 - database_connection_pool_limit: 10 - # background_health_checks: true - # use_shared_health_check: true - # health_check_interval: 30 - # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy - - pass_through_endpoints: - - path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server - target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to - headers: # headers to forward to this URL - content-type: application/json # (Optional) Extra Headers to pass to this endpoint - accept: application/json - forward_headers: True - -# environment_variables: - # settings for using redis caching - # REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com - # REDIS_PORT: "16337" - # REDIS_PASSWORD: \ No newline at end of file +files_settings: + - custom_llm_provider: openai diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index d2a3a6007a0..de6b540a84c 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -1836,132 +1836,6 @@ def test_function_setup_empty_metadata_falls_back_to_litellm_metadata(): -def test_function_setup_litellm_metadata_populates_metadata(): - """ - Test that function_setup() properly handles litellm_metadata (used by /v1/messages, - /batches, /responses, /files endpoints) and populates litellm_params["metadata"] - so callbacks like Langfuse can read API key fields. - - This is the root cause of: Claude Code requests missing user_api_key_hash in Langfuse. - """ - import litellm - - test_api_key_hash = "sk-hashed-1234567890abcdef" - test_team_id = "team-test-123" - test_key_alias = "my-test-key" - - # Simulate what happens for /v1/messages: metadata is in "litellm_metadata", not "metadata" - kwargs = { - "model": "claude-3-5-sonnet", - "messages": [{"role": "user", "content": "hello"}], - "litellm_call_id": "test-call-id-123", - "litellm_metadata": { - "user_api_key_hash": test_api_key_hash, - "user_api_key_alias": test_key_alias, - "user_api_key_team_id": test_team_id, - "user_api_key_user_id": "user-123", - "user_api_key": test_api_key_hash, - }, - } - - logging_obj, returned_kwargs = litellm.utils.function_setup( - original_function="anthropic_messages", - rules_obj=litellm.utils.Rules(), - start_time=time.time(), - **kwargs, - ) - - # litellm_params["metadata"] must contain the API key fields - litellm_params = logging_obj.model_call_details.get("litellm_params", {}) - metadata = litellm_params.get("metadata") - assert metadata is not None, "litellm_params['metadata'] should not be None" - assert isinstance(metadata, dict), "litellm_params['metadata'] should be a dict" - assert metadata.get("user_api_key_hash") == test_api_key_hash - assert metadata.get("user_api_key_alias") == test_key_alias - assert metadata.get("user_api_key_team_id") == test_team_id - - # litellm_metadata should also be preserved - litellm_metadata = litellm_params.get("litellm_metadata") - assert litellm_metadata is not None - assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash - - # metadata should be a COPY, not an alias — mutating one must not affect the other - assert ( - metadata is not litellm_metadata - ), "litellm_params['metadata'] should be a copy, not the same object" - - -def test_function_setup_metadata_takes_precedence_over_litellm_metadata(): - """ - Test that when BOTH metadata and litellm_metadata are present (e.g., user sets - Anthropic API metadata AND proxy adds litellm_metadata), metadata is used as - litellm_params["metadata"] and litellm_metadata is stored separately. - """ - import litellm - - kwargs = { - "model": "claude-3-5-sonnet", - "messages": [{"role": "user", "content": "hello"}], - "litellm_call_id": "test-call-id-456", - "metadata": { - "user_id": "anthropic-user-id", - }, - "litellm_metadata": { - "user_api_key_hash": "sk-hashed-xyz", - "user_api_key_team_id": "team-xyz", - }, - } - - logging_obj, _ = litellm.utils.function_setup( - original_function="anthropic_messages", - rules_obj=litellm.utils.Rules(), - start_time=time.time(), - **kwargs, - ) - - litellm_params = logging_obj.model_call_details.get("litellm_params", {}) - - # When both are present, metadata should be the explicit "metadata" dict - metadata = litellm_params.get("metadata") - assert metadata is not None - assert metadata.get("user_id") == "anthropic-user-id" - - # litellm_metadata should be preserved separately for merge_litellm_metadata() - litellm_metadata = litellm_params.get("litellm_metadata") - assert litellm_metadata is not None - assert litellm_metadata.get("user_api_key_hash") == "sk-hashed-xyz" - - -def test_function_setup_empty_metadata_falls_back_to_litellm_metadata(): - """ - Test that when metadata is explicitly set to {} (empty dict), litellm_metadata - is still used to populate litellm_params["metadata"] so API key fields are visible. - """ - import litellm - - kwargs = { - "model": "claude-3-5-sonnet", - "messages": [{"role": "user", "content": "hello"}], - "litellm_call_id": "test-call-id-789", - "metadata": {}, - "litellm_metadata": { - "user_api_key_hash": "sk-hashed-empty-test", - "user_api_key_team_id": "team-empty-test", - }, - } - - logging_obj, _ = litellm.utils.function_setup( - original_function="anthropic_messages", - rules_obj=litellm.utils.Rules(), - start_time=time.time(), - **kwargs, - ) - - litellm_params = logging_obj.model_call_details.get("litellm_params", {}) - metadata = litellm_params.get("metadata") - assert metadata is not None - assert metadata.get("user_api_key_hash") == "sk-hashed-empty-test" - assert metadata.get("user_api_key_team_id") == "team-empty-test" def test_anthropic_messages_standard_logging_prompt_tokens(): """ Test that standard logging payload for anthropic_messages follows Anthropic convention: