From d89152bb2ccce4ab7713091715dd0abc11dd9ce3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 11 Sep 2025 20:04:24 -0700 Subject: [PATCH 1/3] feat(litellm_logging.py): support new litellm debug parameter - `litellm_request_debug` on requests enables printing raw request when flag is set to true on requests --- litellm/constants.py | 10 +++-- .../litellm_core_utils/get_litellm_params.py | 2 + litellm/litellm_core_utils/litellm_logging.py | 45 +++++++++++++------ litellm/main.py | 44 ++++++++++++------ litellm/types/utils.py | 3 +- 5 files changed, 73 insertions(+), 31 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 75c25d9ea9e..c0ce0f265b5 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -15,7 +15,7 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int( os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) ) DEFAULT_NUM_WORKERS_LITELLM_PROXY = int( - os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", os.cpu_count() or 4) + os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1) ) DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) SQS_SEND_MESSAGE_ACTION = "SendMessage" @@ -60,7 +60,9 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO = int( os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128) ) DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512) + os.getenv( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512 + ) ) # Generic fallback for unknown models @@ -949,7 +951,9 @@ LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token" DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job" PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics" CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data" -CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)) +CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int( + os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000) +) SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup" SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index c354dea0241..c167c202e5d 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -62,6 +62,7 @@ def get_litellm_params( use_litellm_proxy: Optional[bool] = None, api_version: Optional[str] = None, max_retries: Optional[int] = None, + litellm_request_debug: Optional[bool] = None, **kwargs, ) -> dict: litellm_params = { @@ -118,5 +119,6 @@ def get_litellm_params( "vertex_credentials": kwargs.get("vertex_credentials"), "vertex_project": kwargs.get("vertex_project"), "use_litellm_proxy": use_litellm_proxy, + "litellm_request_debug": litellm_request_debug, } return litellm_params diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 19d7c5512ba..59dd09f2728 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -245,6 +245,7 @@ class Logging(LiteLLMLoggingBaseClass): global supabaseClient, promptLayerLogger, weightsBiasesLogger, logfireLogger, capture_exception, add_breadcrumb, lunaryLogger, logfireLogger, prometheusLogger, slack_app custom_pricing: bool = False stream_options = None + litellm_request_debug: bool = False def __init__( self, @@ -470,6 +471,7 @@ class Logging(LiteLLMLoggingBaseClass): **self.litellm_params, **scrub_sensitive_keys_in_metadata(litellm_params), } + self.litellm_request_debug = litellm_params.get("litellm_request_debug", False) self.logger_fn = litellm_params.get("logger_fn", None) verbose_logger.debug(f"self.optional_params: {self.optional_params}") @@ -907,13 +909,19 @@ class Logging(LiteLLMLoggingBaseClass): Prints the RAW curl command sent from LiteLLM """ - if _is_debugging_on(): + if _is_debugging_on() or self.litellm_request_debug: if json_logs: masked_headers = self._get_masked_headers(headers) - verbose_logger.debug( - "POST Request Sent from LiteLLM", - extra={"api_base": {api_base}, **masked_headers}, - ) + if self.litellm_request_debug: + verbose_logger.warning( # .warning ensures this shows up in all environments + "POST Request Sent from LiteLLM", + extra={"api_base": {api_base}, **masked_headers}, + ) + else: + verbose_logger.debug( + "POST Request Sent from LiteLLM", + extra={"api_base": {api_base}, **masked_headers}, + ) else: headers = additional_args.get("headers", {}) if headers is None: @@ -926,7 +934,12 @@ class Logging(LiteLLMLoggingBaseClass): additional_args=additional_args, data=data, ) - verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n") + if self.litellm_request_debug: + verbose_logger.warning( + f"\033[92m{curl_command}\033[0m\n" + ) # .warning ensures this shows up in all environments + else: + verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n") def _get_request_body(self, data: dict) -> str: return str(data) @@ -1714,12 +1727,16 @@ class Logging(LiteLLMLoggingBaseClass): response_obj=result, start_time=start_time, end_time=end_time, - litellm_call_id=current_call_id - if ( - current_call_id := litellm_params.get("litellm_call_id") - ) - is not None - else str(uuid.uuid4()), + litellm_call_id=( + current_call_id + if ( + current_call_id := litellm_params.get( + "litellm_call_id" + ) + ) + is not None + else str(uuid.uuid4()) + ), print_verbose=print_verbose, ) if callback == "wandb" and weightsBiasesLogger is not None: @@ -3367,6 +3384,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915 return galileo_logger # type: ignore elif logging_integration == "cloudzero": from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger + for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): return callback # type: ignore @@ -3594,6 +3612,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915 return callback elif logging_integration == "cloudzero": from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger + for callback in _in_memory_loggers: if isinstance(callback, CloudZeroLogger): return callback @@ -4504,7 +4523,7 @@ def get_standard_logging_object_payload( def emit_standard_logging_payload(payload: StandardLoggingPayload): if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"): - print(json.dumps(payload, indent=4)) # noqa + print(json.dumps(payload, indent=4)) # noqa def get_standard_logging_metadata( diff --git a/litellm/main.py b/litellm/main.py index d7395eb1457..6c81d3eded9 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -150,9 +150,9 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from .llms.custom_llm import CustomLLM, custom_chat_llm_router from .llms.databricks.embed.handler import DatabricksEmbeddingHandler from .llms.deprecated_providers import aleph_alpha, palm +from .llms.gemini.common_utils import get_api_key_from_env from .llms.groq.chat.handler import GroqChatCompletion from .llms.heroku.chat.transformation import HerokuChatConfig -from .llms.gemini.common_utils import get_api_key_from_env from .llms.huggingface.embedding.handler import HuggingFaceEmbedding from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion from .llms.oci.chat.transformation import OCIChatConfig @@ -358,7 +358,9 @@ async def acompletion( logprobs: Optional[bool] = None, top_logprobs: Optional[int] = None, deployment_id=None, - reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None, + reasoning_effort: Optional[ + Literal["none", "minimal", "low", "medium", "high", "default"] + ] = None, safety_identifier: Optional[str] = None, # set api_base, api_version, api_key base_url: Optional[str] = None, @@ -504,7 +506,9 @@ async def acompletion( } if custom_llm_provider is None: _, custom_llm_provider, _, _ = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None) + model=model, + custom_llm_provider=custom_llm_provider, + api_base=completion_kwargs.get("base_url", None), ) fallbacks = fallbacks or litellm.model_fallbacks @@ -899,7 +903,9 @@ def completion( # type: ignore # noqa: PLR0915 logit_bias: Optional[dict] = None, user: Optional[str] = None, # openai v1.0+ new params - reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None, + reasoning_effort: Optional[ + Literal["none", "minimal", "low", "medium", "high", "default"] + ] = None, response_format: Optional[Union[dict, Type[BaseModel]]] = None, seed: Optional[int] = None, tools: Optional[List] = None, @@ -1116,10 +1122,12 @@ def completion( # type: ignore # noqa: PLR0915 ) if provider_specific_header is not None: - headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers( - provider_specific_header=provider_specific_header, - custom_llm_provider=custom_llm_provider, - )) + headers.update( + ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=provider_specific_header, + custom_llm_provider=custom_llm_provider, + ) + ) if model_response is not None and hasattr(model_response, "_hidden_params"): model_response._hidden_params["custom_llm_provider"] = custom_llm_provider @@ -1325,6 +1333,7 @@ def completion( # type: ignore # noqa: PLR0915 azure_scope=kwargs.get("azure_scope"), max_retries=max_retries, timeout=timeout, + litellm_request_debug=kwargs.get("litellm_request_debug", False), ) cast(LiteLLMLoggingObj, logging).update_environment_variables( model=model, @@ -2712,9 +2721,7 @@ def completion( # type: ignore # noqa: PLR0915 ) api_key = ( - api_key - or litellm.api_key - or get_secret("VERCEL_AI_GATEWAY_API_KEY") + api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") ) vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" @@ -2730,7 +2737,7 @@ def completion( # type: ignore # noqa: PLR0915 vercel_headers.update(_headers) headers = vercel_headers - + ## Load Config config = litellm.VercelAIGatewayConfig.get_config() for k, v in config.items(): @@ -3712,7 +3719,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse: func_with_context = partial(ctx.run, func) _, custom_llm_provider, _, _ = get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None) + model=model, + custom_llm_provider=custom_llm_provider, + api_base=kwargs.get("api_base", None), ) # Await normally @@ -5780,7 +5789,14 @@ async def ahealth_check( input=input or ["test"], ), "audio_speech": lambda: litellm.aspeech( - **{**_filter_model_params(model_params), **({"voice": "alloy"} if "voice" not in _filter_model_params(model_params) else {})}, + **{ + **_filter_model_params(model_params), + **( + {"voice": "alloy"} + if "voice" not in _filter_model_params(model_params) + else {} + ), + }, input=prompt or "test", ), "audio_transcription": lambda: litellm.atranscription( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c6f7098a1a7..6452254d11f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1995,7 +1995,7 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): ] guardrail_request: Optional[dict] guardrail_response: Optional[Union[dict, str, List[dict]]] - guardrail_status: Literal["success", "failure","blocked"] + guardrail_status: Literal["success", "failure", "blocked"] start_time: Optional[float] end_time: Optional[float] duration: Optional[float] @@ -2123,6 +2123,7 @@ all_litellm_params = [ "metadata", "litellm_metadata", "litellm_trace_id", + "litellm_request_debug", "guardrails", "tags", "acompletion", From 48619f2b7bd6a71a86c4af57d151a21d1538d1ed Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 11 Sep 2025 20:07:26 -0700 Subject: [PATCH 2/3] fix(litellm_logging.py): support emitting raw response on self.litellm_request_debug is true Addresses https://github.com/BerriAI/litellm/issues/13814 --- litellm/litellm_core_utils/litellm_logging.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 59dd09f2728..2f21d280899 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -996,8 +996,14 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["additional_args"] = additional_args self.model_call_details["log_event_type"] = "post_api_call" + if self.litellm_request_debug: + attr = "warning" + else: + attr = "debug" + if json_logs: - verbose_logger.debug( + callattr = getattr(verbose_logger, attr) + callattr( "RAW RESPONSE:\n{}\n\n".format( self.model_call_details.get( "original_response", self.model_call_details @@ -1005,7 +1011,8 @@ class Logging(LiteLLMLoggingBaseClass): ), ) else: - print_verbose( + callattr = getattr(verbose_logger, attr) + callattr( "RAW RESPONSE:\n{}\n\n".format( self.model_call_details.get( "original_response", self.model_call_details From c802c472b5598de67126d3c93b75ac08ca498c86 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 11 Sep 2025 20:17:39 -0700 Subject: [PATCH 3/3] docs(debugging.md): document new feature Closes https://github.com/BerriAI/litellm/issues/13814 --- docs/my-website/docs/proxy/debugging.md | 68 +++++++++++++++++++------ 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/docs/my-website/docs/proxy/debugging.md b/docs/my-website/docs/proxy/debugging.md index 5cca6541763..fbcac24a4d6 100644 --- a/docs/my-website/docs/proxy/debugging.md +++ b/docs/my-website/docs/proxy/debugging.md @@ -11,13 +11,13 @@ The proxy also supports json logs. [See here](#json-logs) **via cli** -```bash +```bash showLineNumbers $ litellm --debug ``` **via env** -```python +```python showLineNumbers os.environ["LITELLM_LOG"] = "INFO" ``` @@ -25,25 +25,25 @@ os.environ["LITELLM_LOG"] = "INFO" **via cli** -```bash +```bash showLineNumbers $ litellm --detailed_debug ``` **via env** -```python +```python showLineNumbers os.environ["LITELLM_LOG"] = "DEBUG" ``` ### Debug Logs Run the proxy with `--detailed_debug` to view detailed debug logs -```shell +```shell showLineNumbers litellm --config /path/to/config.yaml --detailed_debug ``` When making requests you should see the POST request sent by LiteLLM to the LLM on the Terminal output -```shell +```shell showLineNumbers POST Request Sent from LiteLLM: curl -X POST \ https://api.openai.com/v1/chat/completions \ @@ -51,25 +51,63 @@ https://api.openai.com/v1/chat/completions \ -d '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "this is a test request, write a short poem"}]}' ``` +## Debug single request + +Pass in `litellm_request_debug=True` in the request body + +```bash showLineNumbers +curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model":"fake-openai-endpoint", + "messages": [{"role": "user","content": "How many r in the word strawberry?"}], + "litellm_request_debug": true +}' +``` + +This will emit the raw request sent by LiteLLM to the API Provider and raw response received from the API Provider for **just** this request in the logs. + + +```bash showLineNumbers +INFO: Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit) +20:14:06 - LiteLLM:WARNING: litellm_logging.py:938 - + +POST Request Sent from LiteLLM: +curl -X POST \ +https://exampleopenaiendpoint-production.up.railway.app/chat/completions \ +-H 'Authorization: Be****ey' -H 'Content-Type: application/json' \ +-d '{'model': 'fake', 'messages': [{'role': 'user', 'content': 'How many r in the word strawberry?'}], 'stream': False}' + + +20:14:06 - LiteLLM:WARNING: litellm_logging.py:1015 - RAW RESPONSE: +{"id":"chatcmpl-817fc08f0d6c451485d571dab39b26a1","object":"chat.completion","created":1677652288,"model":"gpt-3.5-turbo-0301","system_fingerprint":"fp_44709d6fcb","choices":[{"index":0,"message":{"role":"assistant","content":"\n\nHello there, how may I assist you today?"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":9,"completion_tokens":12,"total_tokens":21}} + + +INFO: 127.0.0.1:56155 - "POST /chat/completions HTTP/1.1" 200 OK + +``` + + ## JSON LOGS Set `JSON_LOGS="True"` in your env: -```bash +```bash showLineNumbers export JSON_LOGS="True" ``` **OR** Set `json_logs: true` in your yaml: -```yaml +```yaml showLineNumbers litellm_settings: json_logs: true ``` Start proxy -```bash +```bash showLineNumbers $ litellm ``` @@ -80,7 +118,7 @@ The proxy will now all logs in json format. Turn off fastapi's default 'INFO' logs 1. Turn on 'json logs' -```yaml +```yaml showLineNumbers litellm_settings: json_logs: true ``` @@ -89,20 +127,20 @@ litellm_settings: Only get logs if an error occurs. -```bash +```bash showLineNumbers LITELLM_LOG="ERROR" ``` 3. Start proxy -```bash +```bash showLineNumbers $ litellm ``` Expected Output: -```bash +```bash showLineNumbers # no info statements ``` @@ -119,14 +157,14 @@ This can be caused due to all your models hitting rate limit errors, causing the How to control this? - Adjust the cooldown time -```yaml +```yaml showLineNumbers router_settings: cooldown_time: 0 # 👈 KEY CHANGE ``` - Disable Cooldowns [NOT RECOMMENDED] -```yaml +```yaml showLineNumbers router_settings: disable_cooldowns: True ```