diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index 5cb5ab3af2d..b2901650ea6 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -71,17 +71,19 @@ DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source. use to different Send logs through a local DataDog agent (useful for containerized environments): ```shell -DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent -DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) -DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) -DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source +LITELLM_DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent +LITELLM_DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) +DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) +DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source ``` -When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: +When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: - Centralized log shipping in containerized environments - Reducing direct API calls from multiple services - Leveraging agent-side processing and filtering +**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing. + **Step 3**: Start the proxy, make a test request Start proxy @@ -191,8 +193,8 @@ LiteLLM supports customizing the following Datadog environment variables |---------------------|-------------|---------------|----------| | `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* | | `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* | -| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | -| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | +| `LITELLM_DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | +| `LITELLM_DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | | `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | | `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | | `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | @@ -201,5 +203,5 @@ LiteLLM supports customizing the following Datadog environment variables | `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | \* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required -\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required +\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 46e1a2c201f..21e1d562224 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -65,11 +65,11 @@ class DataDogLogger( `DD_SITE` - your datadog site, example = `"us5.datadoghq.com"` Optional environment variables (DataDog Agent): - `DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` - `DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) + `LITELLM_DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` + `LITELLM_DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) - Note: If DD_AGENT_HOST is set, logs will be sent to the agent instead of directly to DataDog API. - In this case, DD_API_KEY and DD_SITE are not required (agent handles authentication). + Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts + with ddtrace which automatically sets DD_AGENT_HOST for APM tracing. """ try: verbose_logger.debug("Datadog: in init datadog logger") @@ -85,7 +85,8 @@ class DataDogLogger( ) # Configure DataDog endpoint (Agent or Direct API) - dd_agent_host = os.getenv("DD_AGENT_HOST") + # Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST + dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST") if dd_agent_host: self._configure_dd_agent(dd_agent_host=dd_agent_host) else: @@ -127,7 +128,7 @@ class DataDogLogger( Args: dd_agent_host: Hostname or IP of DataDog agent """ - dd_agent_port = os.getenv("DD_AGENT_PORT", "10518") # default port for logs + dd_agent_port = os.getenv("LITELLM_DD_AGENT_PORT", "10518") # default port for logs self.intake_url = f"http://{dd_agent_host}:{dd_agent_port}/api/v2/logs" self.DD_API_KEY = os.getenv("DD_API_KEY") # Optional when using agent verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}") diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index 13125aa4952..c877f34ac03 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -633,11 +633,14 @@ async def test_datadog_message_redaction(): def test_datadog_agent_configuration(): """ - Test that DataDog logger correctly configures agent endpoint when DD_AGENT_HOST is set + Test that DataDog logger correctly configures agent endpoint when LITELLM_DD_AGENT_HOST is set. + + Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts + with ddtrace which automatically sets DD_AGENT_HOST for APM tracing. """ test_env = { - "DD_AGENT_HOST": "localhost", - "DD_AGENT_PORT": "10518", + "LITELLM_DD_AGENT_HOST": "localhost", + "LITELLM_DD_AGENT_PORT": "10518", } # Remove DD_SITE and DD_API_KEY to verify they're not required for agent mode @@ -654,4 +657,40 @@ def test_datadog_agent_configuration(): assert dd_logger.intake_url == "http://localhost:10518/api/v2/logs", f"Expected agent URL, got {dd_logger.intake_url}" # Verify DD_API_KEY is optional (can be None) - assert dd_logger.DD_API_KEY is None or isinstance(dd_logger.DD_API_KEY, str) \ No newline at end of file + assert dd_logger.DD_API_KEY is None or isinstance(dd_logger.DD_API_KEY, str) + + +def test_datadog_ignores_ddtrace_agent_host(): + """ + Regression test: Ensure DD_AGENT_HOST set by ddtrace doesn't interfere with LiteLLM logging. + + When users have ddtrace installed for APM tracing, it automatically sets DD_AGENT_HOST. + LiteLLM should ignore DD_AGENT_HOST and only use LITELLM_DD_AGENT_HOST for agent mode. + + This prevents the 404 error when ddtrace's DD_AGENT_HOST points to an APM endpoint + that doesn't support /api/v2/logs. + + Regression test for: https://github.com/BerriAI/litellm/issues/16379 + """ + test_env = { + # User's explicit config for LiteLLM logging (direct API) + "DD_API_KEY": "fake-api-key", + "DD_SITE": "us5.datadoghq.com", + # ddtrace automatically sets these for APM tracing + "DD_AGENT_HOST": "10.176.100.40", + "DD_AGENT_PORT": "8126", + } + + with patch.dict(os.environ, test_env, clear=False): + with patch("asyncio.create_task"): + dd_logger = DataDogLogger() + + # Verify direct API endpoint is used (DD_AGENT_HOST should be ignored) + expected_url = "https://http-intake.logs.us5.datadoghq.com/api/v2/logs" + assert dd_logger.intake_url == expected_url, ( + f"Expected direct API URL '{expected_url}', got '{dd_logger.intake_url}'. " + "DD_AGENT_HOST (set by ddtrace) should be ignored - only LITELLM_DD_AGENT_HOST should trigger agent mode." + ) + + # Verify API key is set correctly + assert dd_logger.DD_API_KEY == "fake-api-key" \ No newline at end of file