mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
Merge branch 'litellm_internal_staging' of https://github.com/BerriAI/litellm into litellm_gemini_latest_cache_read_rates
# Conflicts: # tests/test_litellm/llms/gemini/test_cost_calculator.py
This commit is contained in:
mateo-berri
commit
901bea41e7
28 changed files with 537 additions and 19 deletions
|
|
@ -99,7 +99,7 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportUnknownArgumentType": {
|
||||
"limit": 44530
|
||||
"limit": 44528
|
||||
},
|
||||
"reportUnknownLambdaType": {
|
||||
"limit": 109
|
||||
|
|
@ -138,9 +138,9 @@
|
|||
"limit": 139
|
||||
},
|
||||
"reportUnusedImport": {
|
||||
"limit": 545
|
||||
"limit": 544
|
||||
},
|
||||
"reportUnusedVariable": {
|
||||
"limit": 146
|
||||
"limit": 145
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -591,6 +591,7 @@ def cost_per_token(
|
|||
prompt_characters=prompt_characters,
|
||||
completion_characters=completion_characters,
|
||||
usage=usage_block,
|
||||
service_tier=service_tier,
|
||||
vertex_location=vertex_location,
|
||||
)
|
||||
elif cost_router == "cost_per_token":
|
||||
|
|
@ -845,9 +846,11 @@ def _get_response_model(completion_response: object) -> str | None:
|
|||
_GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: Final[dict] = {
|
||||
# ON_DEMAND_PRIORITY maps to "priority" — selects input_cost_per_token_priority, etc.
|
||||
"ON_DEMAND_PRIORITY": "priority",
|
||||
# FLEX / BATCH maps to "flex" — selects input_cost_per_token_flex, etc.
|
||||
# FLEX / BATCH / ON_DEMAND_FLEX maps to "flex" — selects input_cost_per_token_flex, etc.
|
||||
# Vertex AI reports flex/shared-capacity traffic as ON_DEMAND_FLEX, not FLEX.
|
||||
"FLEX": "flex",
|
||||
"BATCH": "flex",
|
||||
"ON_DEMAND_FLEX": "flex",
|
||||
# ON_DEMAND is standard pricing — no service_tier suffix applied
|
||||
"ON_DEMAND": None,
|
||||
}
|
||||
|
|
@ -862,9 +865,9 @@ def _map_traffic_type_to_service_tier(traffic_type: str | None) -> str | None:
|
|||
|
||||
trafficType values seen in practice
|
||||
------------------------------------
|
||||
ON_DEMAND -> standard pricing (service_tier = None)
|
||||
ON_DEMAND_PRIORITY -> priority pricing (service_tier = "priority")
|
||||
FLEX / BATCH -> batch/flex pricing (service_tier = "flex")
|
||||
ON_DEMAND -> standard pricing (service_tier = None)
|
||||
ON_DEMAND_PRIORITY -> priority pricing (service_tier = "priority")
|
||||
FLEX / BATCH / ON_DEMAND_FLEX -> batch/flex pricing (service_tier = "flex")
|
||||
"""
|
||||
if traffic_type is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -8,6 +8,14 @@ if TYPE_CHECKING:
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
||||
def _completion_response_cost(model_response: "ModelResponse") -> float | None:
|
||||
hidden_params: Final = getattr(model_response, "_hidden_params", None)
|
||||
if not isinstance(hidden_params, dict):
|
||||
return None
|
||||
response_cost: Final = hidden_params.get("response_cost")
|
||||
return response_cost if isinstance(response_cost, float) else None
|
||||
|
||||
|
||||
class SpeechToCompletionBridgeTransformationHandler:
|
||||
def transform_request(
|
||||
self,
|
||||
|
|
@ -123,4 +131,6 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
|
||||
# Create an httpx.Response object
|
||||
response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers)
|
||||
return HttpxBinaryResponseContent(response)
|
||||
binary_response: Final = HttpxBinaryResponseContent(response)
|
||||
binary_response.set_response_cost(_completion_response_cost(model_response))
|
||||
return binary_response
|
||||
|
|
|
|||
|
|
@ -220,6 +220,12 @@
|
|||
"ui_name": "Host URL",
|
||||
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
|
||||
"required": false
|
||||
},
|
||||
"langfuse_environment": {
|
||||
"type": "text",
|
||||
"ui_name": "Tracing Environment",
|
||||
"description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"description": "Langfuse v2 Logging Integration"
|
||||
|
|
@ -247,6 +253,12 @@
|
|||
"ui_name": "Host URL",
|
||||
"description": "Langfuse host URL (default: https://cloud.langfuse.com)",
|
||||
"required": false
|
||||
},
|
||||
"langfuse_environment": {
|
||||
"type": "text",
|
||||
"ui_name": "Tracing Environment",
|
||||
"description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
|
||||
"required": false
|
||||
}
|
||||
},
|
||||
"description": "Langfuse v3 OTEL Logging Integration"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#### What this does ####
|
||||
# On success, logs events to Langfuse
|
||||
import inspect
|
||||
import os
|
||||
import traceback
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
|
|
@ -21,6 +22,9 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
reconstruct_model_name,
|
||||
safe_deep_copy,
|
||||
)
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
validate_langfuse_environment_value,
|
||||
)
|
||||
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
|
|
@ -140,6 +144,7 @@ class LangFuseLogger:
|
|||
langfuse_public_key=None,
|
||||
langfuse_secret=None,
|
||||
langfuse_host=None,
|
||||
langfuse_environment: str | None = None,
|
||||
flush_interval=1,
|
||||
allow_env_credentials: bool = True,
|
||||
):
|
||||
|
|
@ -159,6 +164,10 @@ class LangFuseLogger:
|
|||
if not (self.langfuse_host.startswith("http://") or self.langfuse_host.startswith("https://")):
|
||||
# add http:// if unset, assume communicating over private network - e.g. render
|
||||
self.langfuse_host = "http://" + self.langfuse_host
|
||||
_env_override: Final = str(langfuse_environment).strip() if langfuse_environment is not None else None
|
||||
self.langfuse_environment = _env_override or os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
|
||||
if self.langfuse_environment:
|
||||
validate_langfuse_environment_value(self.langfuse_environment)
|
||||
self.langfuse_release = os.getenv("LANGFUSE_RELEASE")
|
||||
self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
|
||||
self.langfuse_flush_interval = LangFuseLogger._get_langfuse_flush_interval(flush_interval)
|
||||
|
|
@ -182,6 +191,8 @@ class LangFuseLogger:
|
|||
}
|
||||
self.langfuse_sdk_version: str = langfuse.version.__version__
|
||||
|
||||
if "environment" in inspect.signature(Langfuse.__init__).parameters:
|
||||
parameters["environment"] = self.langfuse_environment
|
||||
if Version(self.langfuse_sdk_version) >= Version("2.6.0"):
|
||||
parameters["sdk_integration"] = "litellm"
|
||||
self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import os
|
||||
|
||||
"""
|
||||
This file contains the LangFuseHandler class
|
||||
|
||||
|
|
@ -108,6 +110,7 @@ class LangFuseHandler:
|
|||
langfuse_public_key=credentials.get("langfuse_public_key"),
|
||||
langfuse_secret=credentials.get("langfuse_secret") or credentials.get("langfuse_secret_key"),
|
||||
langfuse_host=credentials.get("langfuse_host"),
|
||||
langfuse_environment=credentials.get("langfuse_environment"),
|
||||
allow_env_credentials=credentials.get("langfuse_host") is None,
|
||||
)
|
||||
in_memory_dynamic_logger_cache.set_cache(
|
||||
|
|
@ -135,8 +138,29 @@ class LangFuseHandler:
|
|||
or standard_callback_dynamic_params.get("langfuse_secret_key"),
|
||||
langfuse_public_key=standard_callback_dynamic_params.get("langfuse_public_key"),
|
||||
langfuse_host=standard_callback_dynamic_params.get("langfuse_host"),
|
||||
langfuse_environment=LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _meaningful_dynamic_environment(
|
||||
standard_callback_dynamic_params: StandardCallbackDynamicParams,
|
||||
) -> str | None:
|
||||
"""Return the per-request environment only when it changes behavior.
|
||||
|
||||
Empty/whitespace values and values equal to the deployment-wide
|
||||
LANGFUSE_TRACING_ENVIRONMENT fallback are treated as absent so an
|
||||
environment-only override that matches the default does not mint a
|
||||
duplicate SDK client (each client costs threads and counts against
|
||||
MAX_LANGFUSE_INITIALIZED_CLIENTS).
|
||||
"""
|
||||
raw = standard_callback_dynamic_params.get("langfuse_environment")
|
||||
if raw is None:
|
||||
return None
|
||||
value = str(raw).strip()
|
||||
if not value or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT"):
|
||||
return None
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _dynamic_langfuse_credentials_are_passed(
|
||||
standard_callback_dynamic_params: StandardCallbackDynamicParams,
|
||||
|
|
@ -153,6 +177,7 @@ class LangFuseHandler:
|
|||
or standard_callback_dynamic_params.get("langfuse_public_key") is not None
|
||||
or standard_callback_dynamic_params.get("langfuse_secret") is not None
|
||||
or standard_callback_dynamic_params.get("langfuse_secret_key") is not None
|
||||
or LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params) is not None
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -231,7 +231,10 @@ class LangfuseOtelLogger(OpenTelemetry):
|
|||
from litellm.integrations.arize._utils import safe_set_attribute
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
|
||||
langfuse_environment: Final = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
|
||||
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
|
||||
langfuse_environment: Final = (
|
||||
dynamic_params.get("langfuse_environment") if dynamic_params else None
|
||||
) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
|
||||
if langfuse_environment:
|
||||
safe_set_attribute(
|
||||
span,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from typing import Any, Final
|
||||
|
||||
|
|
@ -45,12 +46,29 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
|
|||
_raise_env_reference_error(param, source=source)
|
||||
|
||||
|
||||
# Langfuse rejects events whose environment does not match this pattern
|
||||
# (lowercase alphanumerics, hyphens, underscores; no "langfuse" prefix).
|
||||
# Validating here fails fast at config/init time instead of silently
|
||||
# dropping every trace server-side.
|
||||
LANGFUSE_ENVIRONMENT_PATTERN: Final = r"^(?!langfuse)[a-z0-9-_]+$"
|
||||
|
||||
|
||||
def validate_langfuse_environment_value(value: str) -> None:
|
||||
if not re.match(LANGFUSE_ENVIRONMENT_PATTERN, value):
|
||||
raise ValueError(
|
||||
f"Invalid langfuse_environment {value!r}: must be lowercase "
|
||||
"alphanumerics/hyphens/underscores and must not start with "
|
||||
f"'langfuse' (pattern {LANGFUSE_ENVIRONMENT_PATTERN})"
|
||||
)
|
||||
|
||||
|
||||
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
|
||||
_supported_callback_params: Final[tuple[str, ...]] = (
|
||||
"langfuse_public_key",
|
||||
"langfuse_secret",
|
||||
"langfuse_secret_key",
|
||||
"langfuse_host",
|
||||
"langfuse_environment",
|
||||
"langfuse_prompt_version",
|
||||
"langsmith_api_key",
|
||||
"langsmith_project",
|
||||
|
|
|
|||
|
|
@ -1590,7 +1590,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
if transformed_result is not None:
|
||||
result = transformed_result
|
||||
|
||||
if isinstance(result, BaseModel) and hasattr(result, "_hidden_params"):
|
||||
if isinstance(result, (BaseModel, HttpxBinaryResponseContent)) and hasattr(result, "_hidden_params"):
|
||||
hidden_params: Final = getattr(result, "_hidden_params", {})
|
||||
if (
|
||||
"response_cost" in hidden_params and hidden_params["response_cost"] is not None
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ def cost_per_character(
|
|||
usage: Usage,
|
||||
prompt_characters: float | None = None,
|
||||
completion_characters: float | None = None,
|
||||
service_tier: str | None = None,
|
||||
vertex_location: str | None = None,
|
||||
) -> tuple[float, float]:
|
||||
"""
|
||||
|
|
@ -74,6 +75,8 @@ def cost_per_character(
|
|||
- custom_llm_provider: str, "vertex_ai-*"
|
||||
- prompt_characters: float, the number of input characters
|
||||
- completion_characters: float, the number of output characters
|
||||
- service_tier: optional tier derived from Gemini trafficType
|
||||
("priority" for ON_DEMAND_PRIORITY, "flex" for FLEX/batch).
|
||||
- vertex_location: the Vertex AI location serving the request; non-global
|
||||
locations apply the model's regional-endpoint uplift multiplier
|
||||
|
||||
|
|
@ -92,6 +95,7 @@ def cost_per_character(
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
else:
|
||||
try:
|
||||
|
|
@ -123,6 +127,7 @@ def cost_per_character(
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
## CALCULATE OUTPUT COST
|
||||
|
|
@ -131,6 +136,7 @@ def cost_per_character(
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
else:
|
||||
completion_tokens: Final = usage.completion_tokens
|
||||
|
|
@ -162,6 +168,7 @@ def cost_per_character(
|
|||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
usage=usage,
|
||||
service_tier=service_tier,
|
||||
)
|
||||
|
||||
vertex_uplift: Final = get_vertex_regional_endpoint_uplift(model_info, vertex_location)
|
||||
|
|
|
|||
|
|
@ -8013,7 +8013,7 @@ def speech(
|
|||
|
||||
if max_retries is None:
|
||||
max_retries = litellm.num_retries or openai.DEFAULT_MAX_RETRIES
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
litellm_params_dict: Final = get_litellm_params(metadata=metadata, api_key=api_key or dynamic_api_key, **kwargs)
|
||||
|
||||
# Get provider-specific text-to-speech config and map parameters
|
||||
text_to_speech_provider_config = ProviderConfigManager.get_provider_text_to_speech_config(
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
validate_langfuse_environment_value,
|
||||
validate_no_callback_env_reference,
|
||||
)
|
||||
from litellm.types.integrations.compression_interception import (
|
||||
|
|
@ -2027,6 +2028,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase):
|
|||
raise ValueError(f"Invalid callback variable: {key}. Must be one of {valid_keys}")
|
||||
callback_vars[key] = str(value)
|
||||
validate_no_callback_env_reference(key, callback_vars[key], source="key/team callback metadata")
|
||||
if key == "langfuse_environment":
|
||||
validate_langfuse_environment_value(callback_vars[key])
|
||||
return values
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -14,11 +14,36 @@ _NEWRELIC_VAR_PREFIX: Final = "newrelic_"
|
|||
|
||||
|
||||
def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None:
|
||||
if callback_name != _NEWRELIC_CALLBACK or not callback_vars:
|
||||
if not callback_vars:
|
||||
return None
|
||||
env_error: Final = _langfuse_environment_error(callback_vars)
|
||||
if env_error is not None:
|
||||
return env_error
|
||||
if callback_name != _NEWRELIC_CALLBACK:
|
||||
return None
|
||||
return _newrelic_config_error(callback_vars)
|
||||
|
||||
|
||||
def _langfuse_environment_error(callback_vars: Mapping[str, str]) -> str | None:
|
||||
"""Reject langfuse_environment values Langfuse ingestion would drop.
|
||||
|
||||
Accepting an invalid value here would 200 the config write and then
|
||||
silently lose every trace for that key/team at request time.
|
||||
"""
|
||||
value: Final = callback_vars.get("langfuse_environment")
|
||||
if value is None:
|
||||
return None
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
validate_langfuse_environment_value,
|
||||
)
|
||||
|
||||
try:
|
||||
validate_langfuse_environment_value(value)
|
||||
except ValueError as e:
|
||||
return str(e)
|
||||
return None
|
||||
|
||||
|
||||
def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None:
|
||||
"""Validate every ``logging`` entry of a team/key metadata payload."""
|
||||
if not metadata:
|
||||
|
|
|
|||
|
|
@ -262,6 +262,7 @@ async def add_team_callbacks(
|
|||
- langfuse_secret_key: The secret key for the Langfuse callback
|
||||
- langfuse_secret: The secret for the Langfuse callback
|
||||
- langfuse_host: The host for the Langfuse callback
|
||||
- langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)
|
||||
- gcs_bucket_name: The name of the GCS bucket
|
||||
- gcs_path_service_account: The path to the GCS service account
|
||||
- langsmith_api_key: The API key for the Langsmith callback
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
from typing_extensions import TypedDict
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
||||
class LangfuseLoggingConfig(TypedDict):
|
||||
langfuse_secret: str | None
|
||||
langfuse_public_key: str | None
|
||||
langfuse_host: str | None
|
||||
langfuse_environment: ReadOnly[str | None]
|
||||
|
||||
|
||||
class LangfuseUsageDetails(TypedDict):
|
||||
|
|
|
|||
|
|
@ -107,7 +107,17 @@ EmbeddingInput = str | list[str]
|
|||
|
||||
|
||||
class HttpxBinaryResponseContent(_HttpxBinaryResponseContent):
|
||||
_hidden_params: dict = {}
|
||||
_hidden_params: dict
|
||||
|
||||
def __init__(self, response: httpx.Response) -> None:
|
||||
super().__init__(response)
|
||||
self._hidden_params = {} # mutable-ok: mutable-dict contract shared with ModelResponse logging consumers
|
||||
|
||||
def set_response_cost(self, response_cost: float | None) -> None:
|
||||
if response_cost is None:
|
||||
self._hidden_params.pop("response_cost", None)
|
||||
return
|
||||
self._hidden_params["response_cost"] = response_cost
|
||||
|
||||
|
||||
class NotGiven:
|
||||
|
|
|
|||
|
|
@ -3278,6 +3278,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False):
|
|||
langfuse_secret: str | None
|
||||
langfuse_secret_key: str | None
|
||||
langfuse_host: str | None
|
||||
langfuse_environment: ReadOnly[str | None]
|
||||
|
||||
# Langfuse prompt version
|
||||
langfuse_prompt_version: int | None
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@
|
|||
"limit": 3
|
||||
},
|
||||
"F401": {
|
||||
"limit": 14
|
||||
"limit": 13
|
||||
},
|
||||
"LOG015": {
|
||||
"limit": 5
|
||||
|
|
@ -171,7 +171,7 @@
|
|||
"limit": 175
|
||||
},
|
||||
"RUF012": {
|
||||
"limit": 240
|
||||
"limit": 239
|
||||
},
|
||||
"RUF015": {
|
||||
"limit": 8
|
||||
|
|
@ -183,7 +183,7 @@
|
|||
"limit": 4
|
||||
},
|
||||
"RUF059": {
|
||||
"limit": 67
|
||||
"limit": 66
|
||||
},
|
||||
"RUF100": {
|
||||
"limit": 0
|
||||
|
|
|
|||
|
|
@ -1179,6 +1179,14 @@ def test_max_langfuse_clients_limit():
|
|||
class _RecordingLangfuse:
|
||||
last_parameters: Optional[dict] = None
|
||||
|
||||
def __init__(self, environment=None, **parameters):
|
||||
type(self).last_parameters = {"environment": environment, **parameters}
|
||||
self.client = MagicMock()
|
||||
|
||||
|
||||
class _RecordingLangfuseWithoutEnvironment:
|
||||
last_parameters: Optional[dict] = None
|
||||
|
||||
def __init__(self, **parameters):
|
||||
type(self).last_parameters = parameters
|
||||
self.client = MagicMock()
|
||||
|
|
@ -1195,6 +1203,62 @@ def _build_langfuse_logger(monkeypatch) -> LangFuseLogger:
|
|||
)
|
||||
|
||||
|
||||
def test_langfuse_environment_is_passed_to_sdk_client(monkeypatch):
|
||||
monkeypatch.setenv("LANGFUSE_MOCK", "false")
|
||||
monkeypatch.delenv("LANGFUSE_TRACING_ENVIRONMENT", raising=False)
|
||||
monkeypatch.setattr(litellm, "initialized_langfuse_clients", 0)
|
||||
with patch("langfuse.Langfuse", _RecordingLangfuse):
|
||||
logger = LangFuseLogger(
|
||||
langfuse_public_key="pk-env",
|
||||
langfuse_secret="sk-env",
|
||||
langfuse_host="https://test.langfuse.com",
|
||||
langfuse_environment="staging",
|
||||
)
|
||||
assert logger.langfuse_environment == "staging"
|
||||
assert _RecordingLangfuse.last_parameters["environment"] == "staging"
|
||||
|
||||
|
||||
def test_langfuse_environment_falls_back_to_deployment_env_var(monkeypatch):
|
||||
monkeypatch.setenv("LANGFUSE_MOCK", "false")
|
||||
monkeypatch.setenv("LANGFUSE_TRACING_ENVIRONMENT", "deployment-wide")
|
||||
monkeypatch.setattr(litellm, "initialized_langfuse_clients", 0)
|
||||
with patch("langfuse.Langfuse", _RecordingLangfuse):
|
||||
logger = LangFuseLogger(
|
||||
langfuse_public_key="pk-env",
|
||||
langfuse_secret="sk-env",
|
||||
langfuse_host="https://test.langfuse.com",
|
||||
)
|
||||
assert logger.langfuse_environment == "deployment-wide"
|
||||
assert _RecordingLangfuse.last_parameters["environment"] == "deployment-wide"
|
||||
|
||||
|
||||
def test_langfuse_environment_omitted_for_old_sdk_versions(monkeypatch):
|
||||
monkeypatch.setenv("LANGFUSE_MOCK", "false")
|
||||
monkeypatch.setattr(litellm, "initialized_langfuse_clients", 0)
|
||||
with patch("langfuse.Langfuse", _RecordingLangfuseWithoutEnvironment):
|
||||
LangFuseLogger(
|
||||
langfuse_public_key="pk-env",
|
||||
langfuse_secret="sk-env",
|
||||
langfuse_host="https://test.langfuse.com",
|
||||
langfuse_environment="staging",
|
||||
)
|
||||
assert "environment" not in _RecordingLangfuseWithoutEnvironment.last_parameters
|
||||
|
||||
|
||||
def test_dynamic_langfuse_environment_triggers_dynamic_logger():
|
||||
from litellm.integrations.langfuse.langfuse_handler import LangFuseHandler
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
params = StandardCallbackDynamicParams(langfuse_environment="team-a-env")
|
||||
|
||||
assert LangFuseHandler._dynamic_langfuse_credentials_are_passed(params) is True
|
||||
|
||||
config = LangFuseHandler.get_dynamic_langfuse_logging_config(
|
||||
standard_callback_dynamic_params=params
|
||||
)
|
||||
assert config["langfuse_environment"] == "team-a-env"
|
||||
|
||||
|
||||
def test_langfuse_sdk_client_survives_httpx_cache_eviction(monkeypatch):
|
||||
import gc
|
||||
import weakref
|
||||
|
|
@ -1408,3 +1472,52 @@ def test_update_trace_keys_matches_whole_keys_not_substrings():
|
|||
)
|
||||
|
||||
assert "input" not in trace_params
|
||||
|
||||
|
||||
def test_langfuse_environment_is_coerced_and_validated(monkeypatch):
|
||||
monkeypatch.setenv("LANGFUSE_MOCK", "false")
|
||||
monkeypatch.delenv("LANGFUSE_TRACING_ENVIRONMENT", raising=False)
|
||||
monkeypatch.setattr(litellm, "initialized_langfuse_clients", 0)
|
||||
with patch("langfuse.Langfuse", _RecordingLangfuse):
|
||||
logger = LangFuseLogger(
|
||||
langfuse_public_key="pk-env",
|
||||
langfuse_secret="sk-env",
|
||||
langfuse_host="https://test.langfuse.com",
|
||||
langfuse_environment=123, # non-string: must coerce, not crash
|
||||
)
|
||||
assert logger.langfuse_environment == "123"
|
||||
|
||||
with pytest.raises(ValueError, match="langfuse_environment"):
|
||||
LangFuseLogger(
|
||||
langfuse_public_key="pk-env",
|
||||
langfuse_secret="sk-env",
|
||||
langfuse_host="https://test.langfuse.com",
|
||||
langfuse_environment="Production",
|
||||
)
|
||||
|
||||
|
||||
def test_langfuse_empty_environment_falls_back_and_is_not_dynamic(monkeypatch):
|
||||
from litellm.integrations.langfuse.langfuse_handler import LangFuseHandler
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
monkeypatch.setenv("LANGFUSE_TRACING_ENVIRONMENT", "production")
|
||||
|
||||
# '' falls back to the deployment env var at init
|
||||
monkeypatch.setenv("LANGFUSE_MOCK", "false")
|
||||
monkeypatch.setattr(litellm, "initialized_langfuse_clients", 0)
|
||||
with patch("langfuse.Langfuse", _RecordingLangfuse):
|
||||
logger = LangFuseLogger(
|
||||
langfuse_public_key="pk-env",
|
||||
langfuse_secret="sk-env",
|
||||
langfuse_host="https://test.langfuse.com",
|
||||
langfuse_environment="",
|
||||
)
|
||||
assert logger.langfuse_environment == "production"
|
||||
|
||||
# env-only params that add nothing do not select a dynamic logger
|
||||
for redundant in ["", " ", "production"]:
|
||||
params = StandardCallbackDynamicParams(langfuse_environment=redundant)
|
||||
assert LangFuseHandler._dynamic_langfuse_credentials_are_passed(params) is False
|
||||
|
||||
params = StandardCallbackDynamicParams(langfuse_environment="team-a-prod")
|
||||
assert LangFuseHandler._dynamic_langfuse_credentials_are_passed(params) is True
|
||||
|
|
|
|||
|
|
@ -137,6 +137,32 @@ class TestLangfuseOtelIntegration:
|
|||
mock_span, "langfuse.environment", test_env
|
||||
)
|
||||
|
||||
def test_set_langfuse_environment_attribute_prefers_dynamic_param(self):
|
||||
"""Per-key/team langfuse_environment beats the deployment env var."""
|
||||
|
||||
class _RecordingSpan:
|
||||
def __init__(self):
|
||||
self.attributes = {}
|
||||
|
||||
def set_attribute(self, key, value):
|
||||
self.attributes[key] = value
|
||||
|
||||
span = _RecordingSpan()
|
||||
mock_kwargs = {
|
||||
"standard_callback_dynamic_params": {
|
||||
"langfuse_environment": "team-a-env"
|
||||
}
|
||||
}
|
||||
|
||||
with patch.dict(
|
||||
os.environ, {"LANGFUSE_TRACING_ENVIRONMENT": "deployment-wide"}
|
||||
):
|
||||
LangfuseOtelLogger._set_langfuse_specific_attributes(
|
||||
span, mock_kwargs, {}
|
||||
)
|
||||
|
||||
assert span.attributes["langfuse.environment"] == "team-a-env"
|
||||
|
||||
def test_extract_langfuse_metadata_basic(self):
|
||||
"""Ensure metadata is correctly pulled from litellm_params."""
|
||||
metadata_in = {"generation_name": "my-gen", "custom": "data"}
|
||||
|
|
|
|||
|
|
@ -233,3 +233,18 @@ def test_trusted_vars_overlay_uses_shared_parser_semantics():
|
|||
)
|
||||
|
||||
assert params.get("newrelic_api_key") == "12345"
|
||||
|
||||
|
||||
def test_validate_langfuse_environment_value():
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
validate_langfuse_environment_value,
|
||||
)
|
||||
|
||||
validate_langfuse_environment_value("team-a-prod")
|
||||
validate_langfuse_environment_value("staging_2")
|
||||
|
||||
for bad in ["Production", "langfuse-eu", "", "team a"]:
|
||||
with pytest.raises(ValueError, match="langfuse_environment"):
|
||||
validate_langfuse_environment_value(bad)
|
||||
|
|
|
|||
|
|
@ -363,6 +363,36 @@ def test_gemini_image_generation_cost_no_web_search_when_absent(monkeypatch):
|
|||
assert cost_zero == cost_none
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"traffic_type, expected_service_tier",
|
||||
[
|
||||
("ON_DEMAND", None),
|
||||
("ON_DEMAND_PRIORITY", "priority"),
|
||||
("FLEX", "flex"),
|
||||
("BATCH", "flex"),
|
||||
# Vertex AI reports flex/shared-capacity traffic as ON_DEMAND_FLEX.
|
||||
("ON_DEMAND_FLEX", "flex"),
|
||||
# trafficType is matched case-insensitively.
|
||||
("on_demand_flex", "flex"),
|
||||
(None, None),
|
||||
("SOMETHING_UNKNOWN", None),
|
||||
],
|
||||
)
|
||||
def test_map_traffic_type_to_service_tier(
|
||||
traffic_type: str | None, expected_service_tier: str | None
|
||||
):
|
||||
"""
|
||||
Gemini/Vertex usageMetadata.trafficType maps to the LiteLLM service_tier
|
||||
that selects flex/priority cost keys. ON_DEMAND_FLEX (Vertex's flex opt-in
|
||||
value) must map to "flex" so flex-tier requests are not billed as standard.
|
||||
"""
|
||||
from litellm.cost_calculator import _map_traffic_type_to_service_tier
|
||||
|
||||
assert (
|
||||
_map_traffic_type_to_service_tier(traffic_type) == expected_service_tier
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,custom_llm_provider,expected_cache_read_cost",
|
||||
[
|
||||
|
|
@ -378,7 +408,7 @@ def test_flash_alias_cache_read_is_ten_percent_of_input(
|
|||
monkeypatch, model, custom_llm_provider, expected_cache_read_cost
|
||||
):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
model_info = litellm.get_model_info(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
|
|
@ -399,7 +429,7 @@ def test_flash_alias_cache_read_is_ten_percent_of_input(
|
|||
)
|
||||
def test_flash_latest_alias_spellings_price_identically(monkeypatch, prefixed, bare):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
prefixed_entry = litellm.model_cost[prefixed]
|
||||
bare_entry = litellm.model_cost[bare]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
from litellm.proxy.common_utils.callback_config_validation import (
|
||||
callback_config_error,
|
||||
)
|
||||
|
||||
|
||||
def test_callback_config_error_rejects_invalid_langfuse_environment():
|
||||
for callback in ["langfuse", "langfuse_otel"]:
|
||||
error = callback_config_error(callback, {"langfuse_environment": "Production"})
|
||||
assert error is not None and "langfuse_environment" in error
|
||||
|
||||
assert callback_config_error("langfuse", {"langfuse_environment": "team-a-prod"}) is None
|
||||
assert callback_config_error("langfuse", {"langfuse_public_key": "pk"}) is None
|
||||
|
|
@ -2619,6 +2619,49 @@ def test_completion_cost_anthropic_auto_tier_uses_served_priority_rate(_local_mo
|
|||
assert cost == pytest.approx(expected_priority)
|
||||
|
||||
|
||||
def test_completion_cost_vertex_ai_gemini_flex_traffic_type(_local_model_cost_map):
|
||||
"""
|
||||
Vertex AI flex-tier billing regression for issue #37647.
|
||||
|
||||
Vertex Gemini 3.x models route through ``cost_per_character`` (the
|
||||
``cost_router`` token-path gate only matches "gemini-2"), and its token
|
||||
fallbacks dropped ``service_tier``. A response served with
|
||||
``trafficType=ON_DEMAND_FLEX`` must be billed at the flex rate, not the
|
||||
standard rate.
|
||||
"""
|
||||
from litellm import completion_cost
|
||||
|
||||
model = "gemini-3-test-flex-tier-cost-model"
|
||||
litellm.register_model(
|
||||
model_cost={
|
||||
model: {
|
||||
"input_cost_per_token": 1.5e-6,
|
||||
"output_cost_per_token": 9e-6,
|
||||
"input_cost_per_token_flex": 7.5e-7,
|
||||
"output_cost_per_token_flex": 4.5e-6,
|
||||
"litellm_provider": "vertex_ai",
|
||||
"max_tokens": 8192,
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
def _cost_for_traffic_type(traffic_type):
|
||||
usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
response = ModelResponse(usage=usage, model=model)
|
||||
response._hidden_params["provider_specific_fields"] = {"traffic_type": traffic_type}
|
||||
return completion_cost(
|
||||
completion_response=response,
|
||||
model=model,
|
||||
custom_llm_provider="vertex_ai",
|
||||
)
|
||||
|
||||
standard_cost = _cost_for_traffic_type("ON_DEMAND")
|
||||
flex_cost = _cost_for_traffic_type("ON_DEMAND_FLEX")
|
||||
|
||||
assert standard_cost == pytest.approx(1000 * 1.5e-6 + 500 * 9e-6)
|
||||
assert flex_cost == pytest.approx(1000 * 7.5e-7 + 500 * 4.5e-6)
|
||||
|
||||
|
||||
def test_completion_cost_non_string_service_tier_defers_to_served_tier(_local_model_cost_map):
|
||||
"""
|
||||
Regression: a non-string request-level ``service_tier`` (reachable via
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
|
@ -14,6 +19,9 @@ from unittest.mock import MagicMock, patch
|
|||
|
||||
import litellm
|
||||
from litellm import main as litellm_main
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
async def _async_fake_bedrock_image_details(image_url):
|
||||
|
|
@ -2957,3 +2965,109 @@ async def test_acompletion_resolves_provider_from_api_base():
|
|||
)
|
||||
|
||||
assert response.choices[0].message.content == "resolved"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RecordedSpeechSuccess:
|
||||
call_type: str | None
|
||||
spend_metadata: Mapping[str, object]
|
||||
response_cost: float | None
|
||||
logged_response_cost: float | None
|
||||
|
||||
|
||||
def _record_speech_success(payload: dict[str, object]) -> _RecordedSpeechSuccess:
|
||||
call_type: Final = payload.get("call_type")
|
||||
response_cost: Final = payload.get("response_cost")
|
||||
logging_payload: Final = payload.get("standard_logging_object")
|
||||
logged_cost: Final = logging_payload.get("response_cost") if isinstance(logging_payload, dict) else None
|
||||
return _RecordedSpeechSuccess(
|
||||
call_type=call_type if isinstance(call_type, str) else None,
|
||||
spend_metadata=get_litellm_metadata_from_kwargs(payload),
|
||||
response_cost=response_cost if isinstance(response_cost, float) else None,
|
||||
logged_response_cost=logged_cost if isinstance(logged_cost, float) else None,
|
||||
)
|
||||
|
||||
|
||||
class _SuccessEventRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.events: list[_RecordedSpeechSuccess] = [] # mutable-ok: test recorder of success-callback events
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: dict[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.events.append(_record_speech_success(kwargs))
|
||||
|
||||
|
||||
async def _wait_for_success_event(recorder: _SuccessEventRecorder, call_type: str) -> _RecordedSpeechSuccess:
|
||||
for _ in range(100):
|
||||
if (event := next((e for e in recorder.events if e.call_type == call_type), None)) is not None:
|
||||
return event
|
||||
await asyncio.sleep(0.05)
|
||||
pytest.fail(f"no {call_type} success event; got {[e.call_type for e in recorder.events]}")
|
||||
|
||||
|
||||
def _gemini_tts_generate_content_response() -> dict[str, object]:
|
||||
return {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"parts": [
|
||||
{
|
||||
"inlineData": {
|
||||
"mimeType": "audio/L16;codec=pcm;rate=24000",
|
||||
"data": base64.b64encode(b"pcm-audio-bytes").decode(),
|
||||
}
|
||||
}
|
||||
],
|
||||
"role": "model",
|
||||
},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 5,
|
||||
"candidatesTokenCount": 60,
|
||||
"totalTokenCount": 65,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 5}],
|
||||
"candidatesTokensDetails": [{"modality": "AUDIO", "tokenCount": 60}],
|
||||
},
|
||||
"modelVersion": "gemini-2.5-flash-preview-tts",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_aspeech_gemini_bridge_keeps_proxy_metadata_for_spend_tracking(
|
||||
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
|
||||
monkeypatch.delenv("GOOGLE_API_KEY", raising=False)
|
||||
recorder: Final = _SuccessEventRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
mock_route: Final = respx_mock.post(
|
||||
url__regex=r"https://generativelanguage\.googleapis\.com/v1beta/models/gemini-2\.5-flash-preview-tts:generateContent.*"
|
||||
).mock(return_value=httpx.Response(200, json=_gemini_tts_generate_content_response()))
|
||||
|
||||
await litellm.aspeech(
|
||||
model="gemini/gemini-2.5-flash-preview-tts",
|
||||
input="spend tracking check",
|
||||
voice="Kore",
|
||||
api_key="fake-gemini-key",
|
||||
metadata={"user_api_key": "hashed-virtual-key", "user_api_key_user_id": "user-1"},
|
||||
)
|
||||
|
||||
assert mock_route.called
|
||||
assert mock_route.calls.last.request.headers["x-goog-api-key"] == "fake-gemini-key"
|
||||
speech_event: Final = await _wait_for_success_event(recorder, call_type="aspeech")
|
||||
assert speech_event.spend_metadata["user_api_key"] == "hashed-virtual-key"
|
||||
assert speech_event.spend_metadata["user_api_key_user_id"] == "user-1"
|
||||
expected_prompt_cost, expected_completion_cost = litellm.cost_per_token(
|
||||
model="gemini/gemini-2.5-flash-preview-tts",
|
||||
usage_object=Usage(prompt_tokens=5, completion_tokens=60, total_tokens=65),
|
||||
)
|
||||
expected_cost: Final = expected_prompt_cost + expected_completion_cost
|
||||
assert expected_cost > 0
|
||||
assert speech_event.response_cost == pytest.approx(expected_cost)
|
||||
assert speech_event.logged_response_cost == pytest.approx(expected_cost)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import pytest
|
|||
import json
|
||||
|
||||
import litellm
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
|
||||
def test_generic_event():
|
||||
|
|
@ -522,3 +523,34 @@ class TestOpenAIFileObjectBatchGuardrailSerialization:
|
|||
|
||||
page = FileListPage(object="list", data=[self._file_object()], has_more=False)
|
||||
assert "litellm_batch_guardrail" not in page.model_dump(mode="json")["data"][0]
|
||||
|
||||
|
||||
def _binary_content(payload: bytes) -> HttpxBinaryResponseContent:
|
||||
import httpx
|
||||
|
||||
return HttpxBinaryResponseContent(httpx.Response(200, content=payload))
|
||||
|
||||
|
||||
def test_httpx_binary_response_content_hidden_params_are_per_instance():
|
||||
first = _binary_content(b"first")
|
||||
second = _binary_content(b"second")
|
||||
|
||||
first._hidden_params["response_cost"] = 0.5
|
||||
|
||||
assert second._hidden_params == {}
|
||||
|
||||
|
||||
def test_set_response_cost_none_leaves_hidden_params_empty():
|
||||
binary_response = _binary_content(b"audio")
|
||||
|
||||
binary_response.set_response_cost(None)
|
||||
|
||||
assert "response_cost" not in binary_response._hidden_params
|
||||
|
||||
binary_response.set_response_cost(0.25)
|
||||
|
||||
assert binary_response._hidden_params["response_cost"] == 0.25
|
||||
|
||||
binary_response.set_response_cost(None)
|
||||
|
||||
assert "response_cost" not in binary_response._hidden_params
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
|
|||
langfuse_public_key: "text",
|
||||
langfuse_secret_key: "password",
|
||||
langfuse_host: "text",
|
||||
langfuse_environment: "text",
|
||||
},
|
||||
description: "Langfuse v2 Logging Integration",
|
||||
},
|
||||
|
|
@ -121,6 +122,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
|
|||
langfuse_public_key: "text",
|
||||
langfuse_secret_key: "password",
|
||||
langfuse_host: "text",
|
||||
langfuse_environment: "text",
|
||||
},
|
||||
description: "Langfuse v3 OTEL Logging Integration",
|
||||
},
|
||||
|
|
|
|||
1
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
1
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -14836,6 +14836,7 @@ export interface paths {
|
|||
* - langfuse_secret_key: The secret key for the Langfuse callback
|
||||
* - langfuse_secret: The secret for the Langfuse callback
|
||||
* - langfuse_host: The host for the Langfuse callback
|
||||
* - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)
|
||||
* - gcs_bucket_name: The name of the GCS bucket
|
||||
* - gcs_path_service_account: The path to the GCS service account
|
||||
* - langsmith_api_key: The API key for the Langsmith callback
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue