scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Merge pull request #14482 from BerriAI/litellm_dev_09_11_2025_p3

Browse source 
Feature - new `litellm_request_debug=true` flag, enables emitting raw request/response log on single request
This commit is contained in:
Krish Dholakia 2025-09-12 19:56:53 -07:00 • committed by GitHub
commit de8cf40ffa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 128 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
docs/my-website/docs/proxy/debugging.md
View file

@ -11,13 +11,13 @@ The proxy also supports json logs. [See here](#json-logs)
**via cli**
```bash
```bash showLineNumbers
$ litellm --debug
```
**via env**
```python
```python showLineNumbers
os.environ["LITELLM_LOG"] = "INFO"
```
@ -25,25 +25,25 @@ os.environ["LITELLM_LOG"] = "INFO"
**via cli**
```bash
```bash showLineNumbers
$ litellm --detailed_debug
```
**via env**
```python
```python showLineNumbers
os.environ["LITELLM_LOG"] = "DEBUG"
```
### Debug Logs
Run the proxy with `--detailed_debug` to view detailed debug logs
```shell
```shell showLineNumbers
litellm --config /path/to/config.yaml --detailed_debug
```
When making requests you should see the POST request sent by LiteLLM to the LLM on the Terminal output
```shell
```shell showLineNumbers
POST Request Sent from LiteLLM:
curl -X POST \
https://api.openai.com/v1/chat/completions \
@ -51,25 +51,63 @@ https://api.openai.com/v1/chat/completions \
-d '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "this is a test request, write a short poem"}]}'
```
## Debug single request
Pass in `litellm_request_debug=True` in the request body
```bash showLineNumbers
curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model":"fake-openai-endpoint",
"messages": [{"role": "user","content": "How many r in the word strawberry?"}],
"litellm_request_debug": true
}'
```
This will emit the raw request sent by LiteLLM to the API Provider and raw response received from the API Provider for **just** this request in the logs.
```bash showLineNumbers
INFO: Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit)
20:14:06 - LiteLLM:WARNING: litellm_logging.py:938 -
POST Request Sent from LiteLLM:
curl -X POST \
https://exampleopenaiendpoint-production.up.railway.app/chat/completions \
-H 'Authorization: Be****ey' -H 'Content-Type: application/json' \
-d '{'model': 'fake', 'messages': [{'role': 'user', 'content': 'How many r in the word strawberry?'}], 'stream': False}'
20:14:06 - LiteLLM:WARNING: litellm_logging.py:1015 - RAW RESPONSE:
{"id":"chatcmpl-817fc08f0d6c451485d571dab39b26a1","object":"chat.completion","created":1677652288,"model":"gpt-3.5-turbo-0301","system_fingerprint":"fp_44709d6fcb","choices":[{"index":0,"message":{"role":"assistant","content":"\n\nHello there, how may I assist you today?"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":9,"completion_tokens":12,"total_tokens":21}}
INFO: 127.0.0.1:56155 - "POST /chat/completions HTTP/1.1" 200 OK
```
## JSON LOGS
Set `JSON_LOGS="True"` in your env:
```bash
```bash showLineNumbers
export JSON_LOGS="True"
```
**OR**
Set `json_logs: true` in your yaml:
```yaml
```yaml showLineNumbers
litellm_settings:
json_logs: true
```
Start proxy
```bash
```bash showLineNumbers
$ litellm
```
@ -80,7 +118,7 @@ The proxy will now all logs in json format.
Turn off fastapi's default 'INFO' logs
1. Turn on 'json logs'
```yaml
```yaml showLineNumbers
litellm_settings:
json_logs: true
```
@ -89,20 +127,20 @@ litellm_settings:
Only get logs if an error occurs.
```bash
```bash showLineNumbers
LITELLM_LOG="ERROR"
```
3. Start proxy
```bash
```bash showLineNumbers
$ litellm
```
Expected Output:
```bash
```bash showLineNumbers
# no info statements
```
@ -119,14 +157,14 @@ This can be caused due to all your models hitting rate limit errors, causing the
How to control this?
- Adjust the cooldown time
```yaml
```yaml showLineNumbers
router_settings:
cooldown_time: 0 # 👈 KEY CHANGE
```
- Disable Cooldowns [NOT RECOMMENDED]
```yaml
```yaml showLineNumbers
router_settings:
disable_cooldowns: True
```

2
litellm/litellm_core_utils/get_litellm_params.py
View file

@ -62,6 +62,7 @@ def get_litellm_params(
use_litellm_proxy: Optional[bool] = None,
api_version: Optional[str] = None,
max_retries: Optional[int] = None,
litellm_request_debug: Optional[bool] = None,
**kwargs,
) -> dict:
litellm_params = {
@ -118,5 +119,6 @@ def get_litellm_params(
"vertex_credentials": kwargs.get("vertex_credentials"),
"vertex_project": kwargs.get("vertex_project"),
"use_litellm_proxy": use_litellm_proxy,
"litellm_request_debug": litellm_request_debug,
}
return litellm_params

56
litellm/litellm_core_utils/litellm_logging.py
View file

@ -245,6 +245,7 @@ class Logging(LiteLLMLoggingBaseClass):
global supabaseClient, promptLayerLogger, weightsBiasesLogger, logfireLogger, capture_exception, add_breadcrumb, lunaryLogger, logfireLogger, prometheusLogger, slack_app
custom_pricing: bool = False
stream_options = None
litellm_request_debug: bool = False
def __init__(
self,
@ -470,6 +471,7 @@ class Logging(LiteLLMLoggingBaseClass):
**self.litellm_params,
**scrub_sensitive_keys_in_metadata(litellm_params),
}
self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
self.logger_fn = litellm_params.get("logger_fn", None)
verbose_logger.debug(f"self.optional_params: {self.optional_params}")
@ -907,13 +909,19 @@ class Logging(LiteLLMLoggingBaseClass):
Prints the RAW curl command sent from LiteLLM
"""
if _is_debugging_on():
if _is_debugging_on() or self.litellm_request_debug:
if json_logs:
masked_headers = self._get_masked_headers(headers)
verbose_logger.debug(
"POST Request Sent from LiteLLM",
extra={"api_base": {api_base}, **masked_headers},
)
if self.litellm_request_debug:
verbose_logger.warning( # .warning ensures this shows up in all environments
"POST Request Sent from LiteLLM",
extra={"api_base": {api_base}, **masked_headers},
)
else:
verbose_logger.debug(
"POST Request Sent from LiteLLM",
extra={"api_base": {api_base}, **masked_headers},
)
else:
headers = additional_args.get("headers", {})
if headers is None:
@ -926,7 +934,12 @@ class Logging(LiteLLMLoggingBaseClass):
additional_args=additional_args,
data=data,
)
verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
if self.litellm_request_debug:
verbose_logger.warning(
f"\033[92m{curl_command}\033[0m\n"
) # .warning ensures this shows up in all environments
else:
verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
def _get_request_body(self, data: dict) -> str:
return str(data)
@ -983,8 +996,14 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["additional_args"] = additional_args
self.model_call_details["log_event_type"] = "post_api_call"
if self.litellm_request_debug:
attr = "warning"
else:
attr = "debug"
if json_logs:
verbose_logger.debug(
callattr = getattr(verbose_logger, attr)
callattr(
"RAW RESPONSE:\n{}\n\n".format(
self.model_call_details.get(
"original_response", self.model_call_details
@ -992,7 +1011,8 @@ class Logging(LiteLLMLoggingBaseClass):
),
)
else:
print_verbose(
callattr = getattr(verbose_logger, attr)
callattr(
"RAW RESPONSE:\n{}\n\n".format(
self.model_call_details.get(
"original_response", self.model_call_details
@ -1714,12 +1734,16 @@ class Logging(LiteLLMLoggingBaseClass):
response_obj=result,
start_time=start_time,
end_time=end_time,
litellm_call_id=current_call_id
if (
current_call_id := litellm_params.get("litellm_call_id")
)
is not None
else str(uuid.uuid4()),
litellm_call_id=(
current_call_id
if (
current_call_id := litellm_params.get(
"litellm_call_id"
)
)
is not None
else str(uuid.uuid4())
),
print_verbose=print_verbose,
)
if callback == "wandb" and weightsBiasesLogger is not None:
@ -3367,6 +3391,7 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
return galileo_logger # type: ignore
elif logging_integration == "cloudzero":
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
for callback in _in_memory_loggers:
if isinstance(callback, CloudZeroLogger):
return callback # type: ignore
@ -3594,6 +3619,7 @@ def get_custom_logger_compatible_class( # noqa: PLR0915
return callback
elif logging_integration == "cloudzero":
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
for callback in _in_memory_loggers:
if isinstance(callback, CloudZeroLogger):
return callback
@ -4504,7 +4530,7 @@ def get_standard_logging_object_payload(
def emit_standard_logging_payload(payload: StandardLoggingPayload):
if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
print(json.dumps(payload, indent=4)) # noqa
print(json.dumps(payload, indent=4)) # noqa
def get_standard_logging_metadata(

44
litellm/main.py
View file

@ -150,9 +150,9 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from .llms.custom_llm import CustomLLM, custom_chat_llm_router
from .llms.databricks.embed.handler import DatabricksEmbeddingHandler
from .llms.deprecated_providers import aleph_alpha, palm
from .llms.gemini.common_utils import get_api_key_from_env
from .llms.groq.chat.handler import GroqChatCompletion
from .llms.heroku.chat.transformation import HerokuChatConfig
from .llms.gemini.common_utils import get_api_key_from_env
from .llms.huggingface.embedding.handler import HuggingFaceEmbedding
from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion
from .llms.oci.chat.transformation import OCIChatConfig
@ -358,7 +358,9 @@ async def acompletion(
logprobs: Optional[bool] = None,
top_logprobs: Optional[int] = None,
deployment_id=None,
reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
reasoning_effort: Optional[
Literal["none", "minimal", "low", "medium", "high", "default"]
] = None,
safety_identifier: Optional[str] = None,
# set api_base, api_version, api_key
base_url: Optional[str] = None,
@ -504,7 +506,9 @@ async def acompletion(
}
if custom_llm_provider is None:
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None)
model=model,
custom_llm_provider=custom_llm_provider,
api_base=completion_kwargs.get("base_url", None),
)
fallbacks = fallbacks or litellm.model_fallbacks
@ -899,7 +903,9 @@ def completion( # type: ignore # noqa: PLR0915
logit_bias: Optional[dict] = None,
user: Optional[str] = None,
# openai v1.0+ new params
reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
reasoning_effort: Optional[
Literal["none", "minimal", "low", "medium", "high", "default"]
] = None,
response_format: Optional[Union[dict, Type[BaseModel]]] = None,
seed: Optional[int] = None,
tools: Optional[List] = None,
@ -1116,10 +1122,12 @@ def completion( # type: ignore # noqa: PLR0915
)
if provider_specific_header is not None:
headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=provider_specific_header,
custom_llm_provider=custom_llm_provider,
))
headers.update(
ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=provider_specific_header,
custom_llm_provider=custom_llm_provider,
)
)
if model_response is not None and hasattr(model_response, "_hidden_params"):
model_response._hidden_params["custom_llm_provider"] = custom_llm_provider
@ -1325,6 +1333,7 @@ def completion( # type: ignore # noqa: PLR0915
azure_scope=kwargs.get("azure_scope"),
max_retries=max_retries,
timeout=timeout,
litellm_request_debug=kwargs.get("litellm_request_debug", False),
)
cast(LiteLLMLoggingObj, logging).update_environment_variables(
model=model,
@ -2712,9 +2721,7 @@ def completion( # type: ignore # noqa: PLR0915
)
api_key = (
api_key
or litellm.api_key
or get_secret("VERCEL_AI_GATEWAY_API_KEY")
api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
)
vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
@ -2730,7 +2737,7 @@ def completion( # type: ignore # noqa: PLR0915
vercel_headers.update(_headers)
headers = vercel_headers
## Load Config
config = litellm.VercelAIGatewayConfig.get_config()
for k, v in config.items():
@ -3712,7 +3719,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
func_with_context = partial(ctx.run, func)
_, custom_llm_provider, _, _ = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None)
model=model,
custom_llm_provider=custom_llm_provider,
api_base=kwargs.get("api_base", None),
)
# Await normally
@ -5780,7 +5789,14 @@ async def ahealth_check(
input=input or ["test"],
),
"audio_speech": lambda: litellm.aspeech(
**{**_filter_model_params(model_params), **({"voice": "alloy"} if "voice" not in _filter_model_params(model_params) else {})},
**{
**_filter_model_params(model_params),
**(
{"voice": "alloy"}
if "voice" not in _filter_model_params(model_params)
else {}
),
},
input=prompt or "test",
),
"audio_transcription": lambda: litellm.atranscription(

3
litellm/types/utils.py
View file

@ -1996,7 +1996,7 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
]
guardrail_request: Optional[dict]
guardrail_response: Optional[Union[dict, str, List[dict]]]
guardrail_status: Literal["success", "failure","blocked"]
guardrail_status: Literal["success", "failure", "blocked"]
start_time: Optional[float]
end_time: Optional[float]
duration: Optional[float]
@ -2124,6 +2124,7 @@ all_litellm_params = [
"metadata",
"litellm_metadata",
"litellm_trace_id",
"litellm_request_debug",
"guardrails",
"tags",
"acompletion",