import datetime import os import sys import types import unittest from hashlib import sha256 from typing import Optional from unittest.mock import MagicMock, patch import pytest import litellm from litellm.integrations.langfuse import langfuse as langfuse_module from litellm.integrations.langfuse.langfuse import LangFuseLogger sys.path.insert(0, os.path.abspath("../..")) from litellm.integrations.langfuse.langfuse import LangFuseLogger # Import LangfuseUsageDetails directly from the module where it's defined from litellm.types.integrations.langfuse import * class TestLangfuseUsageDetails(unittest.TestCase): def setUp(self): # Save global Langfuse client counter to restore after test self._original_langfuse_clients_count = litellm.initialized_langfuse_clients # Set up environment variables for testing self.env_patcher = patch.dict( "os.environ", { "LANGFUSE_SECRET_KEY": "test-secret-key", "LANGFUSE_PUBLIC_KEY": "test-public-key", "LANGFUSE_HOST": "https://test.langfuse.com", }, ) self.env_patcher.start() # Create mock objects self.mock_langfuse_client = MagicMock() # Mock the client attribute to prevent errors during logger initialization self.mock_langfuse_client.client = MagicMock() self.mock_langfuse_generation = MagicMock() self.mock_langfuse_generation.trace_id = "test-trace-id" self.mock_langfuse_generation.id = "test-generation-id" self.mock_generation_context = MagicMock() self.mock_generation_context.__enter__.return_value = self.mock_langfuse_generation # Setup the trace and generation chain self.last_trace_kwargs = {} def _observation_side_effect(*args, **kwargs): propagated = self.mock_langfuse.propagate_attributes.call_args.kwargs self.last_trace_kwargs = { **kwargs, "id": kwargs["trace_context"]["trace_id"], "session_id": propagated.get("session_id"), } return self.mock_generation_context self.mock_langfuse_client.start_as_current_observation.side_effect = _observation_side_effect self.mock_langfuse_client.create_trace_id.side_effect = lambda seed=None: ( seed or "00000000000000000000000000000001" ) # Mock the langfuse module that's imported locally in methods self.langfuse_module_patcher = patch.dict("sys.modules", {"langfuse": MagicMock()}) self.mock_langfuse_module = self.langfuse_module_patcher.start() # Create a mock for the langfuse module with version self.mock_langfuse = MagicMock() self.mock_langfuse.version = MagicMock() self.mock_langfuse.version.__version__ = "3.0.0" # Set a version that supports all features # Mock the Langfuse class self.mock_langfuse_class = MagicMock() self.mock_langfuse_class.return_value = self.mock_langfuse_client # Set up the sys.modules['langfuse'] mock sys.modules["langfuse"] = self.mock_langfuse sys.modules["langfuse"].Langfuse = self.mock_langfuse_class # Create a fresh logger instance for each test self.logger = LangFuseLogger() # Explicitly set the Langfuse client to our mock self.logger.Langfuse = self.mock_langfuse_client # Ensure langfuse_sdk_version is set correctly for _supports_* methods self.logger.langfuse_sdk_version = "3.0.0" # Add the log_event_on_langfuse method to the instance def log_event_on_langfuse( self, kwargs, response_obj, start_time=None, end_time=None, user_id=None, level="DEFAULT", status_message=None, ): # This implementation calls _log_langfuse_v2 directly return self._log_langfuse_v2( user_id=user_id, metadata=kwargs.get("litellm_params", {}).get("metadata", {}), litellm_params=kwargs.get("litellm_params", {}), output=None, start_time=start_time, end_time=end_time, kwargs=kwargs, optional_params=kwargs.get("optional_params", {}), input=None, response_obj=response_obj, level=level, litellm_call_id=kwargs.get("litellm_call_id", None), ) # Bind the method to the instance self.logger.log_event_on_langfuse = types.MethodType(log_event_on_langfuse, self.logger) # Make sure _is_langfuse_v2 returns True def mock_is_langfuse_v2(self): return True self.logger._is_langfuse_v2 = types.MethodType(mock_is_langfuse_v2, self.logger) def tearDown(self): # Clean up logger instance to prevent state leakage if hasattr(self, "logger"): # Reset logger's Langfuse client to break any references self.logger.Langfuse = None # Delete logger instance to ensure complete cleanup del self.logger # Restore global Langfuse client counter to prevent cross-test pollution litellm.initialized_langfuse_clients = self._original_langfuse_clients_count self.env_patcher.stop() self.langfuse_module_patcher.stop() # patch.dict automatically restores sys.modules def test_langfuse_usage_details_type(self): """Test that LangfuseUsageDetails TypedDict is properly defined with the correct fields""" # Create an instance of LangfuseUsageDetails usage_details: LangfuseUsageDetails = { "input": 10, "output": 20, "total": 30, "cache_creation_input_tokens": 5, "cache_read_input_tokens": 3, } # Verify all fields are present self.assertEqual(usage_details["input"], 10) self.assertEqual(usage_details["output"], 20) self.assertEqual(usage_details["total"], 30) self.assertEqual(usage_details["cache_creation_input_tokens"], 5) self.assertEqual(usage_details["cache_read_input_tokens"], 3) # Test with all fields (all fields are required in TypedDict by default) minimal_usage_details: LangfuseUsageDetails = { "input": 10, "output": 20, "total": 30, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, } self.assertEqual(minimal_usage_details["input"], 10) self.assertEqual(minimal_usage_details["output"], 20) self.assertEqual(minimal_usage_details["total"], 30) def test_log_langfuse_v2_usage_details(self): """Test that usage_details in _log_langfuse_v2 is correctly typed and assigned""" # Create a mock response object with usage information response_obj = MagicMock() response_obj.usage = MagicMock() response_obj.usage.prompt_tokens = 15 response_obj.usage.completion_tokens = 25 # Add the cache token attributes using get method def mock_get(key, default=None): if key == "cache_creation_input_tokens": return 7 elif key == "cache_read_input_tokens": return 4 return default response_obj.usage.get = mock_get # Create kwargs for the log_event method kwargs = { "model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}], "litellm_params": {"metadata": {}}, } # Create start and end times start_time = datetime.datetime.now() end_time = start_time + datetime.timedelta(seconds=1) # Call the log_event method with patch.object(self.logger, "_log_langfuse_v2") as mock_log_langfuse_v2: self.logger.log_event_on_langfuse( kwargs=kwargs, response_obj=response_obj, start_time=start_time, end_time=end_time, ) # Check if _log_langfuse_v2 was called mock_log_langfuse_v2.assert_called_once() # Get the arguments passed to _log_langfuse_v2 call_args = mock_log_langfuse_v2.call_args[1] # Verify response_obj was passed correctly self.assertEqual(call_args["response_obj"], response_obj) def test_create_langfuse_trace_context_normalizes_ids(self): self.mock_langfuse_client.create_trace_id.side_effect = None self.mock_langfuse_client.create_trace_id.return_value = "b" * 32 context = self.logger._create_langfuse_trace_context( trace_id="external-trace-id", parent_observation_id="ABCDEF0123456789", ) assert context == { "trace_id": "b" * 32, "parent_span_id": "abcdef0123456789", } self.mock_langfuse_client.create_trace_id.assert_called_once_with(seed="external-trace-id") parent_observation_id = "550E8400-E29B-41D4-A716-446655440000" context = self.logger._create_langfuse_trace_context( trace_id="a" * 32, parent_observation_id=parent_observation_id, ) assert context == { "trace_id": "a" * 32, "parent_span_id": sha256(parent_observation_id.lower().replace("-", "").encode("utf-8")).digest()[:8].hex(), } def test_langfuse_usage_details_optional_fields(self): """Test that LangfuseUsageDetails fields are properly defined as Optional""" # Create an instance with None values for optional fields usage_details: LangfuseUsageDetails = { "input": 10, "output": 20, "total": 30, "cache_creation_input_tokens": None, "cache_read_input_tokens": None, } # Verify fields can be None self.assertEqual(usage_details["input"], 10) self.assertEqual(usage_details["output"], 20) self.assertEqual(usage_details["total"], 30) self.assertIsNone(usage_details["cache_creation_input_tokens"]) self.assertIsNone(usage_details["cache_read_input_tokens"]) def test_langfuse_usage_details_structure(self): """Test that LangfuseUsageDetails has the correct structure as defined in the commit""" # This test directly verifies the structure of the TypedDict # without relying on the LangFuseLogger class # Create a dictionary that matches the LangfuseUsageDetails structure usage_details = { "input": 15, "output": 25, "total": 40, "cache_creation_input_tokens": 7, "cache_read_input_tokens": 4, } # Verify the structure matches what we expect self.assertIn("input", usage_details) self.assertIn("output", usage_details) self.assertIn("total", usage_details) self.assertIn("cache_creation_input_tokens", usage_details) self.assertIn("cache_read_input_tokens", usage_details) # Verify the values self.assertEqual(usage_details["input"], 15) self.assertEqual(usage_details["output"], 25) self.assertEqual(usage_details["total"], 40) self.assertEqual(usage_details["cache_creation_input_tokens"], 7) self.assertEqual(usage_details["cache_read_input_tokens"], 4) def test_log_langfuse_v2_handles_null_usage_values(self): """ Test that _log_langfuse_v2 correctly handles None values in the usage object by converting them to 0, preventing validation errors. """ # Reset the mock to ensure clean state; clear side_effect so return_value takes effect self.mock_langfuse_client.reset_mock(side_effect=True) self.mock_langfuse_generation.reset_mock(side_effect=True) # Re-setup the trace and generation chain with clean state self.mock_langfuse_generation.id = "test-generation-id" self.mock_generation_context = MagicMock() self.mock_generation_context.__enter__.return_value = self.mock_langfuse_generation self.mock_langfuse_client.start_as_current_observation.return_value = self.mock_generation_context self.mock_langfuse_client.create_trace_id.side_effect = lambda seed=None: ( seed or "00000000000000000000000000000001" ) self.logger.Langfuse = self.mock_langfuse_client with ( patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ) as mock_add_prompt_params, patch.object(self.logger, "_supports_prompt", return_value=True), ): # Create a mock response object with usage information containing None values response_obj = MagicMock() response_obj.usage = MagicMock() response_obj.usage.prompt_tokens = None response_obj.usage.completion_tokens = None response_obj.usage.total_tokens = None # Mock the .get() method to return None for cache-related fields def mock_get(key, default=None): if key in ["cache_creation_input_tokens", "cache_read_input_tokens"]: return None return default response_obj.usage.get = mock_get # Prepare standard kwargs for the call kwargs = { "model": "gpt-4-null-usage", "messages": [{"role": "user", "content": "Test"}], "litellm_params": {"metadata": {}}, "optional_params": {}, "litellm_call_id": "test-call-id-null-usage", "standard_logging_object": None, "response_cost": 0.0, } # Use fixed timestamps to avoid timing-related flakiness fixed_time = datetime.datetime(2024, 1, 1, 12, 0, 0) # Call the method under test try: self.logger._log_langfuse_v2( user_id="test-user", metadata={}, litellm_params=kwargs["litellm_params"], output={"role": "assistant", "content": "Response"}, start_time=fixed_time, end_time=fixed_time + datetime.timedelta(seconds=1), kwargs=kwargs, optional_params=kwargs["optional_params"], input={"messages": kwargs["messages"]}, response_obj=response_obj, level="DEFAULT", litellm_call_id=kwargs["litellm_call_id"], ) except Exception as e: self.fail(f"_log_langfuse_v2 raised an exception: {e}") self.mock_langfuse_client.start_as_current_observation.assert_called() call_args, call_kwargs = self.mock_langfuse_client.start_as_current_observation.call_args usage_details_arg = call_kwargs.get("usage_details") self.assertIsNotNone(usage_details_arg) self.assertEqual(usage_details_arg["input"], 0) self.assertEqual(usage_details_arg["output"], 0) self.assertEqual(usage_details_arg["total"], 0) self.assertEqual(usage_details_arg["cache_creation_input_tokens"], 0) self.assertEqual(usage_details_arg["cache_read_input_tokens"], 0) mock_add_prompt_params.assert_called_once() def _build_standard_logging_payload(self, trace_id: Optional[str] = None): payload = { "id": "payload-id", "call_type": "completion", "response_cost": 0.0, "status": "success", "total_tokens": 0, "prompt_tokens": 0, "completion_tokens": 0, "startTime": 0.0, "endTime": 0.0, "completionStartTime": 0.0, "model": "gpt-4", "model_id": "model-123", "model_group": "openai", "api_base": "https://api.openai.com", "metadata": { "user_api_key_end_user_id": None, "prompt_management_metadata": None, "session_id": None, "trace_name": None, "trace_version": None, "headers": None, "endpoint": None, "caching_groups": None, "previous_models": None, }, "hidden_params": {}, "request_tags": [], "messages": [], "response": {"id": "resp"}, "model_parameters": {}, "guardrail_information": None, "standard_built_in_tools_params": None, } if trace_id is not None: payload["trace_id"] = trace_id return payload def _build_langfuse_kwargs(self, standard_logging_payload): return { "standard_logging_object": standard_logging_payload, "model": standard_logging_payload["model"], "call_type": standard_logging_payload["call_type"], "cache_hit": False, "messages": [], } def test_log_langfuse_v2_uses_standard_trace_id_when_available(self): payload = self._build_standard_logging_payload(trace_id="std-trace-id") kwargs = self._build_langfuse_kwargs(payload) self.last_trace_kwargs = {} with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={}, litellm_params={"metadata": {}}, output=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input=None, response_obj=None, level="INFO", litellm_call_id="call-id-xyz", ) assert self.last_trace_kwargs.get("id") == "std-trace-id" def test_log_langfuse_v2_defaults_to_call_id_without_standard_trace_id(self): payload = self._build_standard_logging_payload() kwargs = self._build_langfuse_kwargs(payload) self.last_trace_kwargs = {} with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={}, litellm_params={"metadata": {}}, output=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input=None, response_obj=None, level="INFO", litellm_call_id="call-id-xyz", ) assert self.last_trace_kwargs.get("id") == "call-id-xyz" def test_log_langfuse_v2_does_not_propagate_tags_to_existing_trace(self): payload = self._build_standard_logging_payload() payload["request_tags"] = ["request-tag"] kwargs = self._build_langfuse_kwargs(payload) with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={"existing_trace_id": "existing-trace-id"}, litellm_params={"metadata": {}}, output=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input=None, response_obj=None, level="INFO", litellm_call_id="call-id-xyz", ) assert self.mock_langfuse.propagate_attributes.call_args.kwargs["tags"] is None def test_log_langfuse_v2_uses_litellm_trace_id_fallback_over_call_id(self): """ When standard_logging_object has no trace_id, but kwargs contains litellm_trace_id (the same ID the DB stores as Session ID), Langfuse should use litellm_trace_id — NOT litellm_call_id. This ensures the trace_id in Langfuse matches the Session ID shown in LiteLLM logs. """ payload = self._build_standard_logging_payload() # no trace_id kwargs = self._build_langfuse_kwargs(payload) kwargs["litellm_trace_id"] = "trace-id-from-kwargs" self.last_trace_kwargs = {} with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={}, litellm_params={"metadata": {}}, output=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input=None, response_obj=None, level="ERROR", litellm_call_id="call-id-xyz", ) # litellm_trace_id should be preferred over litellm_call_id assert self.last_trace_kwargs.get("id") == "trace-id-from-kwargs" def test_log_langfuse_v2_uses_litellm_trace_id_when_standard_logging_object_none( self, ): """ When standard_logging_object is None (failure case where get_standard_logging_object_payload threw), litellm_trace_id from kwargs should be used as the Langfuse trace_id. This matches the DB Session ID. """ kwargs = { "standard_logging_object": None, "model": "gpt-4", "call_type": "completion", "cache_hit": False, "messages": [], "litellm_trace_id": "trace-id-failure", } self.last_trace_kwargs = {} with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={}, litellm_params={"metadata": {}}, output=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input=None, response_obj=None, level="ERROR", litellm_call_id="call-id-different", ) # Must use litellm_trace_id, not litellm_call_id assert self.last_trace_kwargs.get("id") == "trace-id-failure" def test_log_langfuse_v2_session_id_passed_as_trace_session_id(self): """ Test that metadata.session_id is correctly passed as trace_params["session_id"] for Langfuse session grouping, and does NOT override trace_id. Each LLM call should get its own unique trace_id while sharing the session_id. """ payload = self._build_standard_logging_payload(trace_id="std-trace-123") kwargs = self._build_langfuse_kwargs(payload) self.last_trace_kwargs = {} with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={"session_id": "my-session-abc"}, litellm_params={"metadata": {"session_id": "my-session-abc"}}, output=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input=None, response_obj=None, level="INFO", litellm_call_id="call-id-456", ) # session_id should be set for Langfuse session grouping assert self.last_trace_kwargs.get("session_id") == "my-session-abc" # trace_id should remain the standard trace_id, NOT the session_id assert self.last_trace_kwargs.get("id") == "std-trace-123" def test_log_langfuse_v2_session_id_preserved_for_error_level(self): """ Test that session_id is correctly passed in trace_params even when the log level is ERROR (failure case). This verifies the fix for failed requests losing session_id mapping in Langfuse. """ payload = self._build_standard_logging_payload(trace_id="std-trace-err") kwargs = self._build_langfuse_kwargs(payload) self.last_trace_kwargs = {} with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={"session_id": "error-session-xyz"}, litellm_params={"metadata": {"session_id": "error-session-xyz"}}, output="BadRequestError: model not found", start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input={"messages": [{"role": "user", "content": "test"}]}, response_obj=None, level="ERROR", litellm_call_id="call-id-err-789", ) # session_id must be preserved even for ERROR level logs assert self.last_trace_kwargs.get("session_id") == "error-session-xyz" # trace_id should be the standard trace_id, not the session_id assert self.last_trace_kwargs.get("id") == "std-trace-err" # status_message should be set for error traces assert self.last_trace_kwargs.get("status_message") is not None def test_log_langfuse_v2_explicit_trace_id_takes_priority_over_session_id(self): """ Test that when both trace_id and session_id are provided in metadata, trace_id takes priority as the trace identifier. """ payload = self._build_standard_logging_payload() kwargs = self._build_langfuse_kwargs(payload) self.last_trace_kwargs = {} with patch( "litellm.integrations.langfuse.langfuse._add_prompt_to_generation_params", side_effect=lambda generation_params, **kwargs: generation_params, create=True, ): self.logger._log_langfuse_v2( user_id="user-1", metadata={ "session_id": "session-999", "trace_id": "explicit-trace-id-777", }, litellm_params={ "metadata": { "session_id": "session-999", "trace_id": "explicit-trace-id-777", } }, output=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), kwargs=kwargs, optional_params={}, input=None, response_obj=None, level="DEFAULT", litellm_call_id="call-id-aaa", ) # Explicit trace_id must take priority assert self.last_trace_kwargs.get("id") == "explicit-trace-id-777" # session_id must still be set for session grouping assert self.last_trace_kwargs.get("session_id") == "session-999" def test_failure_handler_langfuse_kwargs_excludes_original_response(): """ Test that the actual Logging.failure_handler() passes kwargs without 'original_response' to the Langfuse logger. Exercises the real code path rather than simulating the filtering logic. """ import litellm from litellm.litellm_core_utils.litellm_logging import Logging # Create a Logging instance logging_obj = Logging( model="gpt-4", messages=[{"role": "user", "content": "test"}], stream=False, call_type="completion", start_time=datetime.datetime.utcnow(), litellm_call_id="test-call-id-failure", function_id="test-function-id", ) # Set up model_call_details with original_response (simulates a coroutine) mock_coroutine = MagicMock() logging_obj.model_call_details["original_response"] = mock_coroutine logging_obj.model_call_details["litellm_params"] = { "metadata": {"session_id": "test-session-failure"}, "litellm_session_id": None, } logging_obj.model_call_details["optional_params"] = {} # Capture what gets passed to log_event_on_langfuse captured_kwargs = {} mock_langfuse_logger = MagicMock() def capture_log_event(**log_kwargs): captured_kwargs.update(log_kwargs) return {"trace_id": "mock-trace-id", "generation_id": "mock-gen-id"} mock_langfuse_logger.log_event_on_langfuse.side_effect = capture_log_event # Set "langfuse" as a failure callback so the failure_handler processes it original_failure_callback = litellm.failure_callback litellm.failure_callback = ["langfuse"] try: # Mock LangFuseHandler to return our capturing mock logger with patch("litellm.litellm_core_utils.litellm_logging.LangFuseHandler") as mock_handler_class: mock_handler_class.get_langfuse_logger_for_request.return_value = mock_langfuse_logger # Call the actual failure_handler test_exception = Exception("TestError: model not found") logging_obj.failure_handler( exception=test_exception, traceback_exception="Traceback: test", start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), ) # Verify log_event_on_langfuse was actually called assert mock_langfuse_logger.log_event_on_langfuse.called, "log_event_on_langfuse was not called" # Verify original_response is NOT in the kwargs passed to Langfuse langfuse_kwargs = captured_kwargs.get("kwargs", {}) assert "original_response" not in langfuse_kwargs, ( "original_response should be excluded from kwargs passed to Langfuse" ) # Verify session_id metadata is preserved in the kwargs langfuse_metadata = langfuse_kwargs.get("litellm_params", {}).get("metadata", {}) assert langfuse_metadata.get("session_id") == "test-session-failure", ( "session_id should be preserved in kwargs passed to Langfuse" ) # Verify level is ERROR assert captured_kwargs.get("level") == "ERROR" finally: litellm.failure_callback = original_failure_callback @pytest.mark.asyncio async def test_async_log_failure_event_logs_to_langfuse(): """ Test that LangfusePromptManagement.async_log_failure_event() calls log_event_on_langfuse with level=ERROR even when standard_logging_object is present. This is the code path the proxy uses for failed LLM calls. """ from litellm.integrations.langfuse.langfuse_prompt_management import ( LangfusePromptManagement, ) mock_langfuse_module = MagicMock() mock_langfuse_module.version.__version__ = "3.0.0" with ( patch.dict( "os.environ", { "LANGFUSE_SECRET_KEY": "test-secret", "LANGFUSE_PUBLIC_KEY": "test-public", "LANGFUSE_HOST": "https://test.langfuse.com", }, ), patch.dict("sys.modules", {"langfuse": mock_langfuse_module}), ): prompt_mgmt = LangfusePromptManagement() # Mock the langfuse logger returned by get_langfuse_logger_for_request mock_logger = MagicMock() mock_logger.log_event_on_langfuse.return_value = { "trace_id": "mock-trace", "generation_id": "mock-gen", } with patch("litellm.integrations.langfuse.langfuse_prompt_management.LangFuseHandler") as mock_handler: mock_handler.get_langfuse_logger_for_request.return_value = mock_logger kwargs = { "litellm_params": { "metadata": {"session_id": "test-session-fail"}, }, "litellm_call_id": "call-fail-123", "user": "test-user", "exception": Exception("API error: model not found"), "standard_logging_object": { "error_str": "API error: model not found", "trace_id": "std-trace-fail", "metadata": {}, }, } await prompt_mgmt.async_log_failure_event( kwargs=kwargs, response_obj=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), ) # Verify log_event_on_langfuse was called assert mock_logger.log_event_on_langfuse.called, "log_event_on_langfuse was not called for failure event" call_kwargs = mock_logger.log_event_on_langfuse.call_args[1] assert call_kwargs["level"] == "ERROR" assert call_kwargs["status_message"] == "API error: model not found" assert call_kwargs["response_obj"] is None @pytest.mark.asyncio async def test_async_log_failure_event_works_without_standard_logging_object(): """ Test that async_log_failure_event() still logs to Langfuse even when standard_logging_object is None (e.g. when get_standard_logging_object_payload threw an exception). This is the critical fix — before, it silently returned. """ from litellm.integrations.langfuse.langfuse_prompt_management import ( LangfusePromptManagement, ) mock_langfuse_module = MagicMock() mock_langfuse_module.version.__version__ = "3.0.0" with ( patch.dict( "os.environ", { "LANGFUSE_SECRET_KEY": "test-secret", "LANGFUSE_PUBLIC_KEY": "test-public", "LANGFUSE_HOST": "https://test.langfuse.com", }, ), patch.dict("sys.modules", {"langfuse": mock_langfuse_module}), ): prompt_mgmt = LangfusePromptManagement() mock_logger = MagicMock() mock_logger.log_event_on_langfuse.return_value = { "trace_id": "mock-trace", "generation_id": "mock-gen", } with patch("litellm.integrations.langfuse.langfuse_prompt_management.LangFuseHandler") as mock_handler: mock_handler.get_langfuse_logger_for_request.return_value = mock_logger kwargs = { "litellm_params": { "metadata": {"session_id": "test-session-no-slo"}, }, "litellm_call_id": "call-no-slo-456", "user": "test-user", "exception": Exception("InternalServerError: something broke"), "standard_logging_object": None, # This is the key — it's None } await prompt_mgmt.async_log_failure_event( kwargs=kwargs, response_obj=None, start_time=datetime.datetime.utcnow(), end_time=datetime.datetime.utcnow(), ) # CRITICAL: log_event_on_langfuse MUST still be called assert mock_logger.log_event_on_langfuse.called, ( "log_event_on_langfuse was NOT called when standard_logging_object " "is None — failure trace would be silently dropped" ) call_kwargs = mock_logger.log_event_on_langfuse.call_args[1] assert call_kwargs["level"] == "ERROR" # Falls back to exception from kwargs assert "InternalServerError" in call_kwargs["status_message"] def test_max_langfuse_clients_limit(): """ Test that the max langfuse clients limit is respected when initializing multiple clients """ # Mock langfuse package to avoid triggering real import. # The real langfuse import fails on Python 3.14 due to pydantic v1 incompatibility, # and sys.modules["langfuse"] may be absent after other tests in the suite clean up. mock_langfuse = MagicMock() mock_langfuse.version.__version__ = "3.0.0" # Set max clients to 2 for testing original_initialized_langfuse_clients = litellm.initialized_langfuse_clients with ( patch.dict("sys.modules", {"langfuse": mock_langfuse}), patch.object(langfuse_module, "MAX_LANGFUSE_INITIALIZED_CLIENTS", 2), ): # Reset the counter litellm.initialized_langfuse_clients = 0 # First client should succeed logger1 = LangFuseLogger( langfuse_public_key="test_key_1", langfuse_secret="test_secret_1", langfuse_host="https://test1.langfuse.com", ) assert litellm.initialized_langfuse_clients == 1 # Second client should succeed logger2 = LangFuseLogger( langfuse_public_key="test_key_2", langfuse_secret="test_secret_2", langfuse_host="https://test2.langfuse.com", ) assert litellm.initialized_langfuse_clients == 2 # Third client should fail with exception with pytest.raises(Exception) as exc_info: logger3 = LangFuseLogger( langfuse_public_key="test_key_3", langfuse_secret="test_secret_3", langfuse_host="https://test3.langfuse.com", ) # Verify the error message contains the expected text assert "Max langfuse clients reached" in str(exc_info.value) # Counter should still be 2 (third client failed to initialize) assert litellm.initialized_langfuse_clients == 2 litellm.initialized_langfuse_clients = original_initialized_langfuse_clients