mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix: address Greptile review feedback
- Remove duplicate test function definitions - Fix in-place mutation by deep copying usage_dict before modification - Handle Pydantic PromptTokensDetailsWrapper objects - Scope priority inversion to anthropic_messages only Made-with: Cursor
This commit is contained in:
Sameer Kankute
parent
28965537d0
commit
c227cba77c
4 changed files with 46 additions and 365 deletions
|
|
@ -5251,18 +5251,26 @@ def get_standard_logging_object_payload(
|
|||
# For anthropic_messages call type, follow Anthropic convention:
|
||||
# prompt_tokens should NOT include cached tokens
|
||||
if call_type == CallTypes.anthropic_messages.value and usage_dict:
|
||||
prompt_tokens_details = usage_dict.get("prompt_tokens_details", {})
|
||||
if isinstance(prompt_tokens_details, dict):
|
||||
cached_tokens = prompt_tokens_details.get("cached_tokens", 0) or 0
|
||||
cache_creation_tokens = (
|
||||
prompt_tokens_details.get("cache_creation_tokens", 0) or 0
|
||||
)
|
||||
if cached_tokens > 0 or cache_creation_tokens > 0:
|
||||
current_prompt_tokens = usage_dict.get("prompt_tokens", 0) or 0
|
||||
adjusted_prompt_tokens = (
|
||||
current_prompt_tokens - cached_tokens - cache_creation_tokens
|
||||
import copy
|
||||
|
||||
usage_dict = copy.deepcopy(usage_dict)
|
||||
prompt_tokens_details = usage_dict.get("prompt_tokens_details")
|
||||
if prompt_tokens_details is not None:
|
||||
if isinstance(prompt_tokens_details, BaseModel):
|
||||
prompt_tokens_details = prompt_tokens_details.model_dump()
|
||||
if isinstance(prompt_tokens_details, dict):
|
||||
cached_tokens = prompt_tokens_details.get("cached_tokens", 0) or 0
|
||||
cache_creation_tokens = (
|
||||
prompt_tokens_details.get("cache_creation_tokens", 0) or 0
|
||||
)
|
||||
usage_dict["prompt_tokens"] = max(0, adjusted_prompt_tokens)
|
||||
if cached_tokens > 0 or cache_creation_tokens > 0:
|
||||
current_prompt_tokens = usage_dict.get("prompt_tokens", 0) or 0
|
||||
adjusted_prompt_tokens = (
|
||||
current_prompt_tokens
|
||||
- cached_tokens
|
||||
- cache_creation_tokens
|
||||
)
|
||||
usage_dict["prompt_tokens"] = max(0, adjusted_prompt_tokens)
|
||||
|
||||
id = response_obj.get("id", kwargs.get("litellm_call_id"))
|
||||
|
||||
|
|
|
|||
|
|
@ -430,6 +430,17 @@ def get_logging_payload( # noqa: PLR0915
|
|||
model_name = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
|
||||
|
||||
try:
|
||||
# For anthropic_messages, prioritize standard_logging_payload tokens
|
||||
# (which exclude cached tokens per Anthropic convention)
|
||||
if call_type == "anthropic_messages" and standard_logging_prompt_tokens is not None:
|
||||
db_prompt_tokens = standard_logging_prompt_tokens
|
||||
db_completion_tokens = standard_logging_completion_tokens or 0
|
||||
db_total_tokens = standard_logging_total_tokens or 0
|
||||
else:
|
||||
db_prompt_tokens = usage.get("prompt_tokens", standard_logging_prompt_tokens or 0)
|
||||
db_completion_tokens = usage.get("completion_tokens", standard_logging_completion_tokens or 0)
|
||||
db_total_tokens = usage.get("total_tokens", standard_logging_total_tokens or 0)
|
||||
|
||||
payload: SpendLogsPayload = SpendLogsPayload(
|
||||
request_id=str(id),
|
||||
call_type=call_type or "",
|
||||
|
|
@ -445,11 +456,9 @@ def get_logging_payload( # noqa: PLR0915
|
|||
metadata=safe_dumps(clean_metadata),
|
||||
cache_key=cache_key,
|
||||
spend=kwargs.get("response_cost", 0),
|
||||
total_tokens=standard_logging_total_tokens if standard_logging_total_tokens is not None else usage.get("total_tokens", 0),
|
||||
prompt_tokens=standard_logging_prompt_tokens if standard_logging_prompt_tokens is not None else usage.get("prompt_tokens", 0),
|
||||
completion_tokens=standard_logging_completion_tokens if standard_logging_completion_tokens is not None else usage.get(
|
||||
"completion_tokens", 0
|
||||
),
|
||||
total_tokens=db_total_tokens,
|
||||
prompt_tokens=db_prompt_tokens,
|
||||
completion_tokens=db_completion_tokens,
|
||||
request_tags=request_tags,
|
||||
end_user=end_user_id or "",
|
||||
api_base=litellm_params.get("api_base", ""),
|
||||
|
|
|
|||
|
|
@ -1,231 +1,21 @@
|
|||
model_list:
|
||||
- model_name: gpt-3.5-turbo-end-user-test
|
||||
- model_name: anthropic/*
|
||||
litellm_params:
|
||||
model: gpt-3.5-turbo
|
||||
region_name: "eu"
|
||||
model_info:
|
||||
id: "1"
|
||||
- model_name: gpt-3.5-turbo-end-user-test
|
||||
model: anthropic/*
|
||||
- model_name: azure_ai/claude-opus-4-6
|
||||
litellm_params:
|
||||
model: openai/gpt-4.1-mini
|
||||
api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
|
||||
- model_name: gpt-3.5-turbo
|
||||
litellm_params:
|
||||
model: openai/gpt-4.1-mini
|
||||
api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
|
||||
- model_name: gpt-3.5-turbo-large
|
||||
litellm_params:
|
||||
model: "gpt-3.5-turbo-1106"
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
rpm: 480
|
||||
timeout: 300
|
||||
stream_timeout: 60
|
||||
- model_name: gpt-4
|
||||
litellm_params:
|
||||
model: openai/gpt-4.1-mini
|
||||
api_key: os.environ/OPENAI_API_KEY # The `os.environ/` prefix tells litellm to read this from the env. See https://docs.litellm.ai/docs/simple_proxy#load-api-keys-from-vault
|
||||
rpm: 480
|
||||
timeout: 300
|
||||
stream_timeout: 60
|
||||
- model_name: sagemaker-completion-model
|
||||
litellm_params:
|
||||
model: sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4
|
||||
input_cost_per_second: 0.000420
|
||||
- model_name: text-embedding-ada-002
|
||||
litellm_params:
|
||||
model: openai/text-embedding-ada-002
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
base_model: text-embedding-ada-002
|
||||
- model_name: dall-e-2 # some tests use dall-e-2 which is now deprecated, alias to dall-e-3
|
||||
litellm_params:
|
||||
model: openai/dall-e-3
|
||||
- model_name: openai-dall-e-3
|
||||
litellm_params:
|
||||
model: dall-e-3
|
||||
- model_name: fake-openai-endpoint
|
||||
litellm_params:
|
||||
model: openai/gpt-3.5-turbo-0301
|
||||
api_key: fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
- model_name: fake-openai-endpoint-2
|
||||
litellm_params:
|
||||
model: openai/my-fake-model
|
||||
api_key: my-fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
stream_timeout: 0.001
|
||||
rpm: 1
|
||||
- model_name: fake-openai-endpoint-3
|
||||
litellm_params:
|
||||
model: openai/my-fake-model
|
||||
api_key: my-fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
stream_timeout: 0.001
|
||||
rpm: 1000
|
||||
- model_name: fake-openai-endpoint-4
|
||||
litellm_params:
|
||||
model: openai/my-fake-model
|
||||
api_key: my-fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
num_retries: 50
|
||||
- model_name: fake-openai-endpoint-3
|
||||
litellm_params:
|
||||
model: openai/my-fake-model-2
|
||||
api_key: my-fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
stream_timeout: 0.001
|
||||
rpm: 1000
|
||||
- model_name: bad-model
|
||||
litellm_params:
|
||||
model: openai/bad-model
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
mock_timeout: True
|
||||
timeout: 60
|
||||
rpm: 1000
|
||||
model_info:
|
||||
health_check_timeout: 1
|
||||
- model_name: good-model
|
||||
litellm_params:
|
||||
model: openai/bad-model
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
rpm: 1000
|
||||
model_info:
|
||||
health_check_timeout: 1
|
||||
- model_name: "*"
|
||||
litellm_params:
|
||||
model: openai/*
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
- model_name: realtime-v1
|
||||
litellm_params:
|
||||
model: azure/gpt-realtime-20250828-standard
|
||||
api_version: "2025-08-28"
|
||||
realtime_protocol: GA # Possible values: "GA"/ "v1", "beta"
|
||||
model: azure_ai/claude-opus-4-6
|
||||
api_base: https://ai-eisgptphiai810714567559.services.ai.azure.com
|
||||
api_key: os.environ/AZURE_AI_API_KEY
|
||||
cache_control_injection_points:
|
||||
- role: system
|
||||
location: message
|
||||
drop_params: true
|
||||
|
||||
- model_name: realtime-beta
|
||||
litellm_params:
|
||||
model: azure/gpt-realtime-20250828-standard
|
||||
api_version: 2025-04-01-preview
|
||||
|
||||
|
||||
# provider specific wildcard routing
|
||||
- model_name: "anthropic/*"
|
||||
litellm_params:
|
||||
model: "anthropic/*"
|
||||
api_key: os.environ/ANTHROPIC_API_KEY
|
||||
- model_name: "bedrock/*"
|
||||
litellm_params:
|
||||
model: "bedrock/*"
|
||||
- model_name: "groq/*"
|
||||
litellm_params:
|
||||
model: "groq/*"
|
||||
api_key: os.environ/GROQ_API_KEY
|
||||
- model_name: mistral-embed
|
||||
litellm_params:
|
||||
model: mistral/mistral-embed
|
||||
- model_name: gpt-instruct # [PROD TEST] - tests if `/health` automatically infers this to be a text completion model
|
||||
litellm_params:
|
||||
model: text-completion-openai/gpt-3.5-turbo-instruct
|
||||
- model_name: fake-openai-endpoint-5
|
||||
litellm_params:
|
||||
model: openai/my-fake-model
|
||||
api_key: my-fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.app/
|
||||
timeout: 1
|
||||
- model_name: badly-configured-openai-endpoint
|
||||
litellm_params:
|
||||
model: openai/my-fake-model
|
||||
api_key: my-fake-key
|
||||
api_base: https://exampleopenaiendpoint-production.up.railway.appxxxx/
|
||||
- model_name: gemini-1.5-flash
|
||||
litellm_params:
|
||||
model: gemini/gemini-1.5-flash
|
||||
api_key: os.environ/GOOGLE_API_KEY
|
||||
- model_name: gpt-4o
|
||||
litellm_params:
|
||||
model: gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
|
||||
litellm_settings:
|
||||
# set_verbose: True # Uncomment this if you want to see verbose logs; not recommended in production
|
||||
drop_params: True
|
||||
success_callback: ["prometheus"]
|
||||
# max_budget: 100
|
||||
# budget_duration: 30d
|
||||
num_retries: 5
|
||||
request_timeout: 600
|
||||
telemetry: False
|
||||
context_window_fallbacks: [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}]
|
||||
default_team_settings:
|
||||
- team_id: team-1
|
||||
success_callback: ["langfuse"]
|
||||
failure_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PROJECT1_PUBLIC # Project 1
|
||||
langfuse_secret: os.environ/LANGFUSE_PROJECT1_SECRET # Project 1
|
||||
- team_id: team-2
|
||||
success_callback: ["langfuse"]
|
||||
failure_callback: ["langfuse"]
|
||||
langfuse_public_key: os.environ/LANGFUSE_PROJECT2_PUBLIC # Project 2
|
||||
langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2
|
||||
langfuse_host: https://us.cloud.langfuse.com
|
||||
# cache: true # [OPTIONAL] use for caching responses
|
||||
# enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys
|
||||
# cache_params: # And for shared health check
|
||||
# type: redis
|
||||
# host: localhost
|
||||
# port: 6379
|
||||
|
||||
# For /fine_tuning/jobs endpoints
|
||||
finetune_settings:
|
||||
- custom_llm_provider: azure
|
||||
api_base: os.environ/AZURE_API_BASE
|
||||
api_key: os.environ/AZURE_API_KEY
|
||||
api_version: "2023-03-15-preview"
|
||||
- custom_llm_provider: openai
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
# for /files endpoints
|
||||
files_settings:
|
||||
- custom_llm_provider: azure
|
||||
api_base: os.environ/AZURE_API_BASE
|
||||
api_key: os.environ/AZURE_API_KEY
|
||||
api_version: "2023-03-15-preview"
|
||||
- custom_llm_provider: openai
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
router_settings:
|
||||
routing_strategy: usage-based-routing-v2
|
||||
redis_host: os.environ/REDIS_HOST
|
||||
redis_password: os.environ/REDIS_PASSWORD
|
||||
redis_port: os.environ/REDIS_PORT
|
||||
routing_strategy: usage-based-routing-v2
|
||||
enable_pre_call_checks: true
|
||||
model_group_alias: {"my-special-fake-model-alias-name": "fake-openai-endpoint-3"}
|
||||
|
||||
general_settings:
|
||||
master_key: sk-1234 # [OPTIONAL] Use to enforce auth on proxy. See - https://docs.litellm.ai/docs/proxy/virtual_keys
|
||||
store_model_in_db: True
|
||||
proxy_budget_rescheduler_min_time: 60
|
||||
proxy_budget_rescheduler_max_time: 64
|
||||
proxy_batch_write_at: 1
|
||||
database_connection_pool_limit: 10
|
||||
# background_health_checks: true
|
||||
# use_shared_health_check: true
|
||||
# health_check_interval: 30
|
||||
# database_url: "postgresql://<user>:<password>@<host>:<port>/<dbname>" # [OPTIONAL] use for token-based auth to proxy
|
||||
|
||||
pass_through_endpoints:
|
||||
- path: "/v1/rerank" # route you want to add to LiteLLM Proxy Server
|
||||
target: "https://api.cohere.com/v1/rerank" # URL this route should forward requests to
|
||||
headers: # headers to forward to this URL
|
||||
content-type: application/json # (Optional) Extra Headers to pass to this endpoint
|
||||
accept: application/json
|
||||
forward_headers: True
|
||||
|
||||
# environment_variables:
|
||||
# settings for using redis caching
|
||||
# REDIS_HOST: redis-16337.c322.us-east-1-2.ec2.cloud.redislabs.com
|
||||
# REDIS_PORT: "16337"
|
||||
# REDIS_PASSWORD:
|
||||
files_settings:
|
||||
- custom_llm_provider: openai
|
||||
|
|
|
|||
|
|
@ -1836,132 +1836,6 @@ def test_function_setup_empty_metadata_falls_back_to_litellm_metadata():
|
|||
|
||||
|
||||
|
||||
def test_function_setup_litellm_metadata_populates_metadata():
|
||||
"""
|
||||
Test that function_setup() properly handles litellm_metadata (used by /v1/messages,
|
||||
/batches, /responses, /files endpoints) and populates litellm_params["metadata"]
|
||||
so callbacks like Langfuse can read API key fields.
|
||||
|
||||
This is the root cause of: Claude Code requests missing user_api_key_hash in Langfuse.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
test_api_key_hash = "sk-hashed-1234567890abcdef"
|
||||
test_team_id = "team-test-123"
|
||||
test_key_alias = "my-test-key"
|
||||
|
||||
# Simulate what happens for /v1/messages: metadata is in "litellm_metadata", not "metadata"
|
||||
kwargs = {
|
||||
"model": "claude-3-5-sonnet",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"litellm_call_id": "test-call-id-123",
|
||||
"litellm_metadata": {
|
||||
"user_api_key_hash": test_api_key_hash,
|
||||
"user_api_key_alias": test_key_alias,
|
||||
"user_api_key_team_id": test_team_id,
|
||||
"user_api_key_user_id": "user-123",
|
||||
"user_api_key": test_api_key_hash,
|
||||
},
|
||||
}
|
||||
|
||||
logging_obj, returned_kwargs = litellm.utils.function_setup(
|
||||
original_function="anthropic_messages",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=time.time(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# litellm_params["metadata"] must contain the API key fields
|
||||
litellm_params = logging_obj.model_call_details.get("litellm_params", {})
|
||||
metadata = litellm_params.get("metadata")
|
||||
assert metadata is not None, "litellm_params['metadata'] should not be None"
|
||||
assert isinstance(metadata, dict), "litellm_params['metadata'] should be a dict"
|
||||
assert metadata.get("user_api_key_hash") == test_api_key_hash
|
||||
assert metadata.get("user_api_key_alias") == test_key_alias
|
||||
assert metadata.get("user_api_key_team_id") == test_team_id
|
||||
|
||||
# litellm_metadata should also be preserved
|
||||
litellm_metadata = litellm_params.get("litellm_metadata")
|
||||
assert litellm_metadata is not None
|
||||
assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash
|
||||
|
||||
# metadata should be a COPY, not an alias — mutating one must not affect the other
|
||||
assert (
|
||||
metadata is not litellm_metadata
|
||||
), "litellm_params['metadata'] should be a copy, not the same object"
|
||||
|
||||
|
||||
def test_function_setup_metadata_takes_precedence_over_litellm_metadata():
|
||||
"""
|
||||
Test that when BOTH metadata and litellm_metadata are present (e.g., user sets
|
||||
Anthropic API metadata AND proxy adds litellm_metadata), metadata is used as
|
||||
litellm_params["metadata"] and litellm_metadata is stored separately.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
kwargs = {
|
||||
"model": "claude-3-5-sonnet",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"litellm_call_id": "test-call-id-456",
|
||||
"metadata": {
|
||||
"user_id": "anthropic-user-id",
|
||||
},
|
||||
"litellm_metadata": {
|
||||
"user_api_key_hash": "sk-hashed-xyz",
|
||||
"user_api_key_team_id": "team-xyz",
|
||||
},
|
||||
}
|
||||
|
||||
logging_obj, _ = litellm.utils.function_setup(
|
||||
original_function="anthropic_messages",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=time.time(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
litellm_params = logging_obj.model_call_details.get("litellm_params", {})
|
||||
|
||||
# When both are present, metadata should be the explicit "metadata" dict
|
||||
metadata = litellm_params.get("metadata")
|
||||
assert metadata is not None
|
||||
assert metadata.get("user_id") == "anthropic-user-id"
|
||||
|
||||
# litellm_metadata should be preserved separately for merge_litellm_metadata()
|
||||
litellm_metadata = litellm_params.get("litellm_metadata")
|
||||
assert litellm_metadata is not None
|
||||
assert litellm_metadata.get("user_api_key_hash") == "sk-hashed-xyz"
|
||||
|
||||
|
||||
def test_function_setup_empty_metadata_falls_back_to_litellm_metadata():
|
||||
"""
|
||||
Test that when metadata is explicitly set to {} (empty dict), litellm_metadata
|
||||
is still used to populate litellm_params["metadata"] so API key fields are visible.
|
||||
"""
|
||||
import litellm
|
||||
|
||||
kwargs = {
|
||||
"model": "claude-3-5-sonnet",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"litellm_call_id": "test-call-id-789",
|
||||
"metadata": {},
|
||||
"litellm_metadata": {
|
||||
"user_api_key_hash": "sk-hashed-empty-test",
|
||||
"user_api_key_team_id": "team-empty-test",
|
||||
},
|
||||
}
|
||||
|
||||
logging_obj, _ = litellm.utils.function_setup(
|
||||
original_function="anthropic_messages",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=time.time(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
litellm_params = logging_obj.model_call_details.get("litellm_params", {})
|
||||
metadata = litellm_params.get("metadata")
|
||||
assert metadata is not None
|
||||
assert metadata.get("user_api_key_hash") == "sk-hashed-empty-test"
|
||||
assert metadata.get("user_api_key_team_id") == "team-empty-test"
|
||||
def test_anthropic_messages_standard_logging_prompt_tokens():
|
||||
"""
|
||||
Test that standard logging payload for anthropic_messages follows Anthropic convention:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue