import gzip import json import os from datetime import datetime from pathlib import Path from typing import Awaitable, Coroutine, Final, TypedDict, TypeVar from unittest.mock import AsyncMock, patch import pytest import respx from httpx import Request, Response from pydantic import TypeAdapter from typing_extensions import ReadOnly import litellm import litellm.integrations.datadog.datadog as datadog_module from litellm.caching.caching import Cache from litellm.caching.llm_caching_handler import LLMClientCache from litellm.integrations.datadog.datadog import DataDogLogger from litellm.integrations.datadog.datadog_handler import ( get_datadog_env, get_datadog_hostname, get_datadog_pod_name, get_datadog_service, get_datadog_source, get_datadog_tags, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.integrations.datadog import DatadogInitParams, DatadogPayload, DataDogStatus from litellm.types.utils import ( StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelInformation, StandardLoggingPayload, ) import asyncio import io from datetime import datetime as datetime_class STANDARD_START_TIME: Final[datetime] = datetime(2025, 1, 1) STANDARD_END_TIME: Final[datetime] = datetime(2025, 1, 1, 0, 0, 1) _T = TypeVar("_T") async def _await_with_spawned_tasks(call: Awaitable[_T]) -> _T: before: Final = asyncio.all_tasks() result: Final = await call spawned: Final = asyncio.all_tasks() - before - {asyncio.current_task()} await asyncio.wait_for(asyncio.gather(*spawned), timeout=5) return result def _discard_periodic_flush(coroutine: Coroutine[object, object, None]) -> None: coroutine.close() class _DummySpan: def __init__(self, trace_id=None, span_id=None): self.trace_id = trace_id self.span_id = span_id class _DummyTracer: def __init__(self, current_span=None, current_root_span=None): self._current_span = current_span self._current_root_span = current_root_span def current_span(self): return self._current_span def current_root_span(self): return self._current_root_span def _standard_logging_payload() -> StandardLoggingPayload: return StandardLoggingPayload( id="test_id", trace_id="trace-id", session_id="session-id", litellm_call_id="call-id", call_type="completion", stream=False, response_cost=0.1, cost_breakdown=None, autorouter_savings=None, autorouter_savings_estimate=None, autorouter_baseline_observation=None, response_cost_failure_debug_info=None, status="success", status_fields={}, custom_llm_provider="openai", total_tokens=30, prompt_tokens=20, completion_tokens=10, startTime=1234567890.0, endTime=1234567891.0, completionStartTime=1234567890.5, response_time=1.0, model_map_information=StandardLoggingModelInformation( model_map_key="gpt-4.1-mini", model_map_value=None, ), model="gpt-4.1-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", user_agent=None, metadata=StandardLoggingMetadata( user_api_key_hash="test_hash", user_api_key_org_id=None, user_api_key_alias="test_alias", user_api_key_team_id="test_team", user_api_key_user_id="test_user", user_api_key_team_alias="test_team_alias", spend_logs_metadata=None, requester_ip_address="127.0.0.1", requester_metadata=None, ), cache_hit=False, cache_key=None, saved_cache_cost=0.0, request_tags=[], end_user=None, requester_ip_address="127.0.0.1", messages=[{"role": "user", "content": "Hello, world!"}], response={"choices": [{"message": {"content": "Hi there!"}}]}, error_str=None, error_information=None, model_parameters={"stream": True}, hidden_params=StandardLoggingHiddenParams( model_id="model-123", cache_key=None, api_base="https://api.openai.com", response_cost="0.1", additional_headers=None, ), guardrail_information=None, standard_built_in_tools_params=None, ) @pytest.fixture def datadog_env(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DD_API_KEY", "test_api_key") monkeypatch.setenv("DD_SITE", "test.datadoghq.com") monkeypatch.delenv("LITELLM_DD_AGENT_HOST", raising=False) monkeypatch.delenv("LITELLM_DD_AGENT_PORT", raising=False) monkeypatch.delenv("DD_BASE_URL", raising=False) monkeypatch.setattr(litellm, "datadog_params", None) monkeypatch.setattr(litellm, "datadog_use_v1", False) @pytest.fixture def datadog_logger(datadog_env: None) -> DataDogLogger: with patch("asyncio.create_task", side_effect=_discard_periodic_flush): logger: Final = DataDogLogger() return logger @pytest.mark.asyncio async def test_add_trace_context_uses_current_span(monkeypatch): monkeypatch.setenv("DD_SITE", "https://fake.datadoghq.com") monkeypatch.setenv("DD_API_KEY", "anything") tracer = _DummyTracer(current_span=_DummySpan(trace_id=123, span_id=456)) monkeypatch.setattr(datadog_module, "tracer", tracer) dd_logger = DataDogLogger() payload = DatadogPayload( ddsource="litellm", ddtags="env:test", hostname="host", message="{}", service="svc", status="info", ) dd_logger._add_trace_context_to_payload(payload) assert payload["dd.trace_id"] == "123" assert payload["dd.span_id"] == "456" @pytest.mark.asyncio async def test_add_trace_context_falls_back_to_root_span(monkeypatch): monkeypatch.setenv("DD_SITE", "https://fake.datadoghq.com") monkeypatch.setenv("DD_API_KEY", "anything") tracer = _DummyTracer( current_span=None, current_root_span=_DummySpan(trace_id=789, span_id=None), ) monkeypatch.setattr(datadog_module, "tracer", tracer) dd_logger = DataDogLogger() payload = DatadogPayload( ddsource="litellm", ddtags="env:test", hostname="host", message="{}", service="svc", status="info", ) dd_logger._add_trace_context_to_payload(payload) assert payload["dd.trace_id"] == "789" assert "dd.span_id" not in payload @pytest.mark.asyncio async def test_add_trace_context_handles_missing_tracer(monkeypatch): monkeypatch.setenv("DD_SITE", "https://fake.datadoghq.com") monkeypatch.setenv("DD_API_KEY", "anything") monkeypatch.setattr(datadog_module, "tracer", object()) dd_logger = DataDogLogger() payload = DatadogPayload( ddsource="litellm", ddtags="env:test", hostname="host", message="{}", service="svc", status="info", ) dd_logger._add_trace_context_to_payload(payload) assert "dd.trace_id" not in payload assert "dd.span_id" not in payload @pytest.mark.asyncio async def test_add_trace_context_ignores_span_without_trace_id(monkeypatch): monkeypatch.setenv("DD_SITE", "https://fake.datadoghq.com") monkeypatch.setenv("DD_API_KEY", "anything") tracer = _DummyTracer(current_span=_DummySpan(trace_id=None, span_id=555)) monkeypatch.setattr(datadog_module, "tracer", tracer) dd_logger = DataDogLogger() payload = DatadogPayload( ddsource="litellm", ddtags="env:test", hostname="host", message="{}", service="svc", status="info", ) dd_logger._add_trace_context_to_payload(payload) assert "dd.trace_id" not in payload assert "dd.span_id" not in payload @pytest.mark.asyncio @pytest.mark.usefixtures("datadog_env", "drained_logging_worker") async def test_datadog_logging_http_request(): """ - Test that the HTTP request is made to Datadog - sent to the /api/v2/logs endpoint - the payload is batched - each element in the payload is a DatadogPayload - each element in a DatadogPayload.message contains all the valid fields """ try: from litellm.integrations.datadog.datadog import DataDogLogger os.environ["DD_SITE"] = "https://fake.datadoghq.com" os.environ["DD_API_KEY"] = "anything" dd_logger = DataDogLogger() litellm.callbacks = [dd_logger] litellm.set_verbose = True # Create a mock for the async_client's post method mock_post = AsyncMock() mock_post.return_value.status_code = 202 mock_post.return_value.text = "Accepted" dd_logger.async_client.post = mock_post # Make the completion call for _ in range(5): response = await _await_with_spawned_tasks( litellm.acompletion( model="gpt-4.1-mini", messages=[{"role": "user", "content": "what llm are u"}], max_tokens=10, temperature=0.2, mock_response="Accepted", ) ) print(response) await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=5) await dd_logger.flush_queue() # Assert that the mock was called assert mock_post.called, "HTTP request was not made" # Get the arguments of the last call args, kwargs = mock_post.call_args print("CAll args and kwargs", args, kwargs) # Print the request body # You can add more specific assertions here if needed # For example, checking if the URL is correct assert kwargs["url"].endswith("/api/v2/logs"), "Incorrect DataDog endpoint" body = kwargs["data"] # use gzip to unzip the body with gzip.open(io.BytesIO(body), "rb") as f: body = f.read().decode("utf-8") print(body) # body is string parse it to dict body = json.loads(body) print(body) assert len(body) == 5 # 5 logs should be sent to DataDog # Assert that the first element in body has the expected fields and shape assert isinstance(body[0], dict), "First element in body should be a dictionary" # Get the expected fields and their types from DatadogPayload expected_fields = DatadogPayload.__annotations__ required_fields = { "ddsource": str, "ddtags": str, "hostname": str, "message": str, "service": str, "status": str, } optional_fields = set(expected_fields.keys()) - set(required_fields.keys()) # Assert that all elements in body have the required fields with correct types for log in body: assert isinstance(log, dict), "Each log should be a dictionary" for field, expected_type in required_fields.items(): assert field in log, f"Field '{field}' is missing from the log" assert isinstance( log[field], expected_type ), f"Field '{field}' has incorrect type. Expected {expected_type}, got {type(log[field])}" for optional_field in optional_fields: if optional_field in log: assert isinstance( log[optional_field], str ), f"Optional field '{optional_field}' must be a string" unexpected_fields = set(log.keys()) - set(expected_fields.keys()) assert ( not unexpected_fields ), f"Log contains unexpected fields: {unexpected_fields}" # Parse the 'message' field as JSON and check its structure message = json.loads(body[0]["message"]) print("logged message", json.dumps(message, indent=4)) expected_message_fields = StandardLoggingPayload.__required_keys__ for field in expected_message_fields: assert field in message, f"Field '{field}' is missing from the message" # Check specific fields assert message["call_type"] == "acompletion" assert message["model"] == "gpt-4.1-mini" assert isinstance(message["model_parameters"], dict) assert "temperature" in message["model_parameters"] assert "max_tokens" in message["model_parameters"] assert isinstance(message["response"], dict) assert isinstance(message["metadata"], dict) except Exception as e: pytest.fail(f"Test failed with exception: {str(e)}") @pytest.mark.asyncio async def test_datadog_payload_environment_variables(): """Test that DataDog payload correctly includes environment variables in the payload structure""" try: # Set test environment variables test_env = { "DD_ENV": "test-env", "DD_SERVICE": "test-service", "DD_VERSION": "1.0.0", "DD_SOURCE": "test-source", "DD_API_KEY": "fake-key", "DD_SITE": "datadoghq.com", } with patch.dict(os.environ, test_env): dd_logger = DataDogLogger() standard_payload = create_standard_logging_payload() # Create the payload dd_payload = dd_logger.create_datadog_logging_payload( kwargs={"standard_logging_object": standard_payload}, response_obj=None, start_time=datetime_class.now(), end_time=datetime_class.now(), ) print("dd payload=", json.dumps(dd_payload, indent=2)) # Verify payload structure and environment variables assert ( dd_payload["ddsource"] == "test-source" ), "Incorrect source in payload" assert ( dd_payload["service"] == "test-service" ), "Incorrect service in payload" assert ( "env:test-env,service:test-service,version:1.0.0,HOSTNAME:" in dd_payload["ddtags"] ), "Incorrect tags in payload" except Exception as e: pytest.fail(f"Test failed with exception: {str(e)}") @pytest.mark.asyncio @pytest.mark.usefixtures("fake_provider_credentials") async def test_datadog_payload_content_truncation(): """ Test that DataDog payload correctly truncates long content DataDog has a limit of 1MB for the logged payload size. """ dd_logger = DataDogLogger() # Create a standard payload with very long content standard_payload = create_standard_logging_payload() long_content = "x" * 80_000 # Create string longer than MAX_STR_LENGTH (10_000) # Modify payload with long content standard_payload["error_str"] = long_content standard_payload["messages"] = [ { "role": "user", "content": [ { "type": "image_url", "image_url": { "url": long_content, "detail": "low", }, } ], } ] standard_payload["response"] = {"choices": [{"message": {"content": long_content}}]} # Create the payload dd_payload = dd_logger.create_datadog_logging_payload( kwargs={"standard_logging_object": standard_payload}, response_obj=None, start_time=datetime_class.now(), end_time=datetime_class.now(), ) print("dd_payload", json.dumps(dd_payload, indent=2)) # Parse the message back to dict to verify truncation message_dict = json.loads(dd_payload["message"]) # Verify truncation of fields assert len(message_dict["error_str"]) < 10_100, "error_str not truncated correctly" assert ( len(str(message_dict["messages"])) < 10_100 ), "messages not truncated correctly" assert ( len(str(message_dict["response"])) < 10_100 ), "response not truncated correctly" @pytest.mark.asyncio async def test_datadog_payload_truncation_leaves_shared_payload_intact(monkeypatch): """ Every callback of a request shares one standard logging object, so the datadog truncation must not turn its messages into a string for the callbacks that run after it (the prompt caching router check reads `messages` as a list to pin the deployment holding the cache) """ monkeypatch.setenv("DD_SITE", "https://fake.datadoghq.com") monkeypatch.setenv("DD_API_KEY", "anything") dd_logger = DataDogLogger() standard_payload = create_standard_logging_payload() original_messages = [{"role": "user", "content": "x" * 80_000}] standard_payload["messages"] = original_messages kwargs = {"standard_logging_object": standard_payload} dd_payload = dd_logger.create_datadog_logging_payload( kwargs=kwargs, response_obj=None, start_time=datetime_class.now(), end_time=datetime_class.now(), ) assert kwargs["standard_logging_object"]["messages"] is original_messages assert len(json.loads(dd_payload["message"])["messages"]) < 10_100 def test_datadog_static_methods(): """Test the static helper methods in DataDogLogger class""" # Test with default environment variables assert get_datadog_source() == "litellm" assert get_datadog_service() == "litellm-server" assert get_datadog_hostname() is not None assert get_datadog_env() == "unknown" assert get_datadog_pod_name() == "unknown" # Test tags format with default values assert "env:unknown,service:litellm-server,version:unknown,HOSTNAME:" in ",".join( get_datadog_tags() ) # Test with custom environment variables test_env = { "DD_SOURCE": "custom-source", "DD_SERVICE": "custom-service", "HOSTNAME": "test-host", "DD_ENV": "production", "DD_VERSION": "1.0.0", "POD_NAME": "pod-123", } with patch.dict(os.environ, test_env): assert get_datadog_source() == "custom-source" print("DataDogLogger._get_datadog_source()", get_datadog_source()) assert get_datadog_service() == "custom-service" print("DataDogLogger._get_datadog_service()", get_datadog_service()) assert get_datadog_hostname() == "test-host" print( "DataDogLogger._get_datadog_hostname()", get_datadog_hostname(), ) assert get_datadog_env() == "production" print("DataDogLogger._get_datadog_env()", get_datadog_env()) assert get_datadog_pod_name() == "pod-123" print( "DataDogLogger._get_datadog_pod_name()", get_datadog_pod_name(), ) # Test tags format with custom values expected_custom_tags = "env:production,service:custom-service,version:1.0.0,HOSTNAME:test-host,POD_NAME:pod-123" print("DataDogLogger._get_datadog_tags()", get_datadog_tags()) assert ",".join(get_datadog_tags()) == expected_custom_tags @pytest.mark.asyncio @pytest.mark.usefixtures("fake_provider_credentials") async def test_datadog_non_serializable_messages(): """Test logging events with non-JSON-serializable messages""" dd_logger = DataDogLogger() # Create payload with non-serializable content standard_payload = create_standard_logging_payload() non_serializable_obj = datetime_class.now() # datetime objects aren't JSON serializable standard_payload["messages"] = [{"role": "user", "content": non_serializable_obj}] standard_payload["response"] = { "choices": [{"message": {"content": non_serializable_obj}}] } kwargs = {"standard_logging_object": standard_payload} # Test payload creation dd_payload = dd_logger.create_datadog_logging_payload( kwargs=kwargs, response_obj=None, start_time=datetime_class.now(), end_time=datetime_class.now(), ) # Verify payload can be serialized assert dd_payload["status"] == DataDogStatus.INFO # Verify the message can be parsed back to dict dict_payload = json.loads(dd_payload["message"]) # Check that the non-serializable objects were converted to strings assert isinstance(dict_payload["messages"][0]["content"], str) assert isinstance(dict_payload["response"]["choices"][0]["message"]["content"], str) def test_get_datadog_tags(): """Test the _get_datadog_tags static method with various inputs""" # Test with no standard_logging_object and default env vars base_tags = get_datadog_tags() assert any("env:" in t for t in base_tags) assert any("service:" in t for t in base_tags) assert any("version:" in t for t in base_tags) assert any("POD_NAME:" in t for t in base_tags) assert any("HOSTNAME:" in t for t in base_tags) # Test with custom env vars test_env = { "DD_ENV": "production", "DD_SERVICE": "custom-service", "DD_VERSION": "1.0.0", "HOSTNAME": "test-host", "POD_NAME": "pod-123", } with patch.dict(os.environ, test_env): custom_tags = get_datadog_tags() assert "env:production" in custom_tags assert "service:custom-service" in custom_tags assert "version:1.0.0" in custom_tags assert "HOSTNAME:test-host" in custom_tags assert "POD_NAME:pod-123" in custom_tags # Test with standard_logging_object containing request_tags standard_logging_obj = create_standard_logging_payload() standard_logging_obj["request_tags"] = ["tag1", "tag2"] tags_with_request = get_datadog_tags(standard_logging_obj) assert "request_tag:tag1" in tags_with_request assert "request_tag:tag2" in tags_with_request # Test with empty request_tags standard_logging_obj["request_tags"] = [] tags_empty_request = get_datadog_tags(standard_logging_obj) assert not any(t.startswith("request_tag:") for t in tags_empty_request) # Test with None request_tags standard_logging_obj["request_tags"] = None tags_none_request = get_datadog_tags(standard_logging_obj) assert not any(t.startswith("request_tag:") for t in tags_none_request) @pytest.mark.asyncio async def test_datadog_message_redaction(): """ Test that DataDog logger correctly initializes with turn_off_message_logging=True from litellm.datadog_params """ try: # Test using litellm.datadog_params pattern litellm.datadog_params = DatadogInitParams(turn_off_message_logging=True) os.environ["DD_SITE"] = "https://fake.datadoghq.com" os.environ["DD_API_KEY"] = "anything" # Mock the periodic flush to avoid async issues with patch("asyncio.create_task"): dd_logger = DataDogLogger() # Verify that turn_off_message_logging was set correctly from litellm.datadog_params assert hasattr( dd_logger, "turn_off_message_logging" ), "DataDogLogger should have turn_off_message_logging attribute" assert ( dd_logger.turn_off_message_logging is True ), f"Expected turn_off_message_logging=True, got {dd_logger.turn_off_message_logging}" # Test the redaction method inherited from CustomLogger model_call_details = { "standard_logging_object": { "messages": [ { "role": "user", "content": "This is sensitive information that should be redacted", } ], "response": { "choices": [ { "message": { "content": "This is a sensitive response that should be redacted" } } ] }, } } # Apply redaction using the inherited method redacted_details = ( dd_logger.redact_standard_logging_payload_from_model_call_details( model_call_details ) ) redacted_str = "redacted-by-litellm" # Verify that messages are redacted redacted_standard_obj = redacted_details["standard_logging_object"] assert ( redacted_standard_obj["messages"][0]["content"] == redacted_str ), f"Messages not redacted. Got: {redacted_standard_obj['messages'][0]['content']}" # Verify that response is redacted assert ( redacted_standard_obj["response"]["choices"][0]["message"]["content"] == redacted_str ), f"Response not redacted. Got: {redacted_standard_obj['response']['choices'][0]['message']['content']}" print("✅ DataDog message redaction test passed") except Exception as e: pytest.fail(f"Test failed with exception: {str(e)}") finally: # Clean up litellm.datadog_params = None litellm.callbacks = [] def test_datadog_agent_configuration(): """ Test that DataDog logger correctly configures agent endpoint when LITELLM_DD_AGENT_HOST is set. Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts with ddtrace which automatically sets DD_AGENT_HOST for APM tracing. """ test_env = { "LITELLM_DD_AGENT_HOST": "localhost", "LITELLM_DD_AGENT_PORT": "10518", } # Remove DD_SITE and DD_API_KEY to verify they're not required for agent mode env_to_remove = ["DD_SITE", "DD_API_KEY"] with patch.dict(os.environ, test_env, clear=False): for key in env_to_remove: os.environ.pop(key, None) with patch("asyncio.create_task"): dd_logger = DataDogLogger() # Verify agent endpoint is configured correctly assert ( dd_logger.intake_url == "http://localhost:10518/api/v2/logs" ), f"Expected agent URL, got {dd_logger.intake_url}" # Verify DD_API_KEY is optional (can be None) assert dd_logger.DD_API_KEY is None or isinstance(dd_logger.DD_API_KEY, str) def test_datadog_ignores_ddtrace_agent_host(): """ Regression test: Ensure DD_AGENT_HOST set by ddtrace doesn't interfere with LiteLLM logging. When users have ddtrace installed for APM tracing, it automatically sets DD_AGENT_HOST. LiteLLM should ignore DD_AGENT_HOST and only use LITELLM_DD_AGENT_HOST for agent mode. This prevents the 404 error when ddtrace's DD_AGENT_HOST points to an APM endpoint that doesn't support /api/v2/logs. Regression test for: https://github.com/BerriAI/litellm/issues/16379 """ test_env = { # User's explicit config for LiteLLM logging (direct API) "DD_API_KEY": "fake-api-key", "DD_SITE": "us5.datadoghq.com", # ddtrace automatically sets these for APM tracing "DD_AGENT_HOST": "10.176.100.40", "DD_AGENT_PORT": "8126", } with patch.dict(os.environ, test_env, clear=False): with patch("asyncio.create_task"): dd_logger = DataDogLogger() # Verify direct API endpoint is used (DD_AGENT_HOST should be ignored) expected_url = "https://http-intake.logs.us5.datadoghq.com/api/v2/logs" assert dd_logger.intake_url == expected_url, ( f"Expected direct API URL '{expected_url}', got '{dd_logger.intake_url}'. " "DD_AGENT_HOST (set by ddtrace) should be ignored - only LITELLM_DD_AGENT_HOST should trigger agent mode." ) # Verify API key is set correctly assert dd_logger.DD_API_KEY == "fake-api-key" def create_standard_logging_payload() -> StandardLoggingPayload: return StandardLoggingPayload( id="test_id", call_type="completion", response_cost=0.1, response_cost_failure_debug_info=None, status="success", total_tokens=30, prompt_tokens=20, completion_tokens=10, startTime=1234567890.0, endTime=1234567891.0, completionStartTime=1234567890.5, model_map_information=StandardLoggingModelInformation( model_map_key="gpt-4.1-mini", model_map_value=None ), model="gpt-4.1-mini", model_id="model-123", model_group="openai-gpt", api_base="https://api.openai.com", metadata=StandardLoggingMetadata( user_api_key_hash="test_hash", user_api_key_org_id=None, user_api_key_alias="test_alias", user_api_key_team_id="test_team", user_api_key_user_id="test_user", user_api_key_team_alias="test_team_alias", spend_logs_metadata=None, requester_ip_address="127.0.0.1", requester_metadata=None, ), cache_hit=False, cache_key=None, saved_cache_cost=0.0, request_tags=[], end_user=None, requester_ip_address="127.0.0.1", messages=[{"role": "user", "content": "Hello, world!"}], response={"choices": [{"message": {"content": "Hi there!"}}]}, error_str=None, model_parameters={"stream": True}, hidden_params=StandardLoggingHiddenParams( model_id="model-123", cache_key=None, api_base="https://api.openai.com", response_cost="0.1", additional_headers=None, ), ) _INTAKE_URL: Final = "https://http-intake.logs.test.datadoghq.com/api/v2/logs" class _ServiceEventMessage(TypedDict): service: ReadOnly[str] call_type: ReadOnly[str] error: ReadOnly[str] is_error: ReadOnly[bool] @pytest.fixture def delivery( datadog_env: None, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter ) -> tuple[DataDogLogger, respx.Route]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) monkeypatch.delenv("DD_SOURCE", raising=False) monkeypatch.delenv("DD_SERVICE", raising=False) with patch("asyncio.create_task", side_effect=_discard_periodic_flush): logger: Final = DataDogLogger() intake: Final = respx_mock.post(_INTAKE_URL).mock(return_value=Response(202, text="Accepted")) return logger, intake def _delivered_logs(intake: respx.Route) -> list[DatadogPayload]: return TypeAdapter(list[DatadogPayload]).validate_json(gzip.decompress(intake.calls.last.request.content)) @pytest.mark.asyncio async def test_a_successful_request_is_delivered_as_an_info_log_carrying_the_standard_payload( delivery: tuple[DataDogLogger, respx.Route], ) -> None: datadog_logger, intake = delivery standard_payload: Final = _standard_logging_payload() await datadog_logger.async_log_success_event( kwargs={"standard_logging_object": standard_payload}, response_obj=None, start_time=STANDARD_START_TIME, end_time=STANDARD_END_TIME, ) await datadog_logger.async_send_batch() assert intake.call_count == 1 logs: Final = _delivered_logs(intake) assert len(logs) == 1 assert logs[0]["ddsource"] == "litellm" assert logs[0]["service"] == "litellm-server" assert logs[0]["status"] == DataDogStatus.INFO assert TypeAdapter(dict[str, object]).validate_json(logs[0]["message"]) == standard_payload @pytest.mark.asyncio async def test_a_failed_request_is_delivered_as_an_error_log_that_keeps_the_error_string( delivery: tuple[DataDogLogger, respx.Route], ) -> None: datadog_logger, intake = delivery standard_payload: Final = _standard_logging_payload() standard_payload["status"] = "failure" standard_payload["error_str"] = "Test error" await datadog_logger.async_log_failure_event( kwargs={"standard_logging_object": standard_payload}, response_obj=None, start_time=STANDARD_START_TIME, end_time=STANDARD_END_TIME, ) await datadog_logger.async_send_batch() assert intake.call_count == 1 logs: Final = _delivered_logs(intake) assert len(logs) == 1 assert logs[0]["status"] == DataDogStatus.ERROR message: Final = TypeAdapter(dict[str, object]).validate_json(logs[0]["message"]) assert message == standard_payload assert message["error_str"] == "Test error" @pytest.mark.asyncio async def test_a_failing_redis_cache_is_delivered_to_datadog_as_redis_warnings( delivery: tuple[DataDogLogger, respx.Route], monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: datadog_logger, intake = delivery absent_socket: Final = str(tmp_path / "absent.sock") redis_cache: Final = Cache(type="redis", url=f"unix://{absent_socket}") monkeypatch.setattr(redis_cache.cache.service_logger_obj, "dd_logger", datadog_logger, raising=False) monkeypatch.setattr(litellm, "service_callback", ["datadog"]) monkeypatch.setattr(litellm, "callbacks", []) monkeypatch.setattr(litellm, "cache", redis_cache) for _ in range(3): await litellm.acompletion( model="gpt-4.1-mini", messages=[{"role": "user", "content": "what llm are u"}], mock_response="Accepted", caching=True, ) await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) await datadog_logger.async_send_batch() assert intake.call_count == 1 logs: Final = _delivered_logs(intake) assert len(logs) > 0 assert {log["status"] for log in logs} == {DataDogStatus.WARN} messages: Final = [TypeAdapter(_ServiceEventMessage).validate_json(log["message"]) for log in logs] assert {message["service"] for message in messages} == {"redis"} assert all(message["is_error"] is True for message in messages) assert all(absent_socket in message["error"] for message in messages)