scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 213

fix(proxy): recognize *.cognitiveservices.azure.com as OpenAI-compatible in pass-through cost tracking (#29730)

Browse source 
* fix(proxy): recognize *.cognitiveservices.azure.com as OpenAI-compatible

Azure OpenAI resources created via the newer "Azure AI Foundry" /
Cognitive Services pathway live on `*.cognitiveservices.azure.com`
subdomains, not the older `openai.azure.com`. Both are valid Azure
OpenAI surfaces in production today.

The OpenAI pass-through cost-tracking handler hard-codes only the older
hostname in five places (four `is_openai_*_route` methods on
OpenAIPassthroughLoggingHandler, plus is_openai_route on
PassThroughEndpointLogging). As a result, calls from newer Azure
deployments are silently classified as "not an OpenAI route", the
dispatch into the cost-tracking handler is skipped, and tokens/cost
never get extracted into LiteLLM_SpendLogs — the row gets written with
prompt_tokens=0, completion_tokens=0, spend=0, model='unknown'.

Reproduced 2026-06-04 against a real Azure OpenAI deployment on
`*.cognitiveservices.azure.com` proxied through LiteLLM v1.88.0.

Fix: factor the hostname check into a single helper
`_is_openai_compatible_host` listing all three recognized surfaces
(api.openai.com, openai.azure.com, cognitiveservices.azure.com), and
have all five call sites delegate to it. Purely additive — never
weakens recognition for the originally-supported hostnames.

Adds a test
`test_is_openai_route_recognizes_cognitiveservices_azure_com` that
exercises all four `is_openai_*_route` static methods against
`*.cognitiveservices.azure.com` URLs (positive cases per route + a
small cross-route negative to confirm route-specific path matching
still works on the new hostname).

Out of scope for this PR (separate followup):
  - `openai_passthrough_handler` calls chat/completions
    `transform_response` on Responses API payloads (`output:` not
    `choices:`), which throws inside the dispatch and drops the
    SpendLogs row entirely. Recognized + tracked separately.

* ci: trigger fresh run

Empty commit to re-run checks. The previous auth-and-jwt failure was
a transient HuggingFace Hub 429 rate-limit hitting tokenizer downloads
in tests/proxy_unit_tests/test_custom_tokenizer_bug.py — unrelated to
this PR's scope (hostname recognition in pass-through cost tracking).
No code change.

---------

Co-authored-by: shin-berri <shin-laptop@berri.ai>
Co-authored-by: yuneng-jiang <yuneng@berri.ai>
This commit is contained in:
Liam Scott 2026-06-08 08:05:59 -04:00 • committed by GitHub
parent 1d6402a89a
commit a30801e083
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 89 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py
View file

@ -32,6 +32,24 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
from litellm.types.utils import ImageResponse, LlmProviders, PassthroughCallTypes
from litellm.utils import ModelResponse, TextCompletionResponse
# Hostnames that route to OpenAI-compatible APIs. `cognitiveservices.azure.com`
# is the Azure subdomain used by newer Azure OpenAI resource types (e.g. the
# "Azure AI Foundry" / Cognitive Services-hosted deployments). Older Azure
# OpenAI resources use `openai.azure.com`; both are valid in production.
_OPENAI_COMPATIBLE_HOSTNAMES = (
"api.openai.com",
"openai.azure.com",
"cognitiveservices.azure.com",
)
def _is_openai_compatible_host(hostname: Optional[str]) -> bool:
"""True if the hostname is one of the recognized OpenAI-compatible
surfaces (OpenAI proper or any Azure OpenAI subdomain)."""
if not hostname:
return False
return any(host in hostname for host in _OPENAI_COMPATIBLE_HOSTNAMES)
class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
"""
@ -52,12 +70,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
if not url_route:
return False
parsed_url = urlparse(url_route)
return bool(
parsed_url.hostname
and (
"api.openai.com" in parsed_url.hostname
or "openai.azure.com" in parsed_url.hostname
)
return (
_is_openai_compatible_host(parsed_url.hostname)
and "/v1/chat/completions" in parsed_url.path
)
@ -67,12 +81,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
if not url_route:
return False
parsed_url = urlparse(url_route)
return bool(
parsed_url.hostname
and (
"api.openai.com" in parsed_url.hostname
or "openai.azure.com" in parsed_url.hostname
)
return (
_is_openai_compatible_host(parsed_url.hostname)
and "/v1/images/generations" in parsed_url.path
)
@ -82,12 +92,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
if not url_route:
return False
parsed_url = urlparse(url_route)
return bool(
parsed_url.hostname
and (
"api.openai.com" in parsed_url.hostname
or "openai.azure.com" in parsed_url.hostname
)
return (
_is_openai_compatible_host(parsed_url.hostname)
and "/v1/images/edits" in parsed_url.path
)
@ -97,13 +103,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
if not url_route:
return False
parsed_url = urlparse(url_route)
return bool(
parsed_url.hostname
and (
"api.openai.com" in parsed_url.hostname
or "openai.azure.com" in parsed_url.hostname
)
and ("/v1/responses" in parsed_url.path or "/responses" in parsed_url.path)
return _is_openai_compatible_host(parsed_url.hostname) and (
"/v1/responses" in parsed_url.path or "/responses" in parsed_url.path
)
def _get_user_from_metadata(

9
litellm/proxy/pass_through_endpoints/success_handler.py
View file

@ -437,12 +437,13 @@ class PassThroughEndpointLogging:
"""Check if the URL route is an OpenAI API route."""
if not url_route:
return False
parsed_url = urlparse(url_route)
return parsed_url.hostname and (
"api.openai.com" in parsed_url.hostname
or "openai.azure.com" in parsed_url.hostname
from .llm_provider_handlers.openai_passthrough_logging_handler import (
_is_openai_compatible_host,
)
parsed_url = urlparse(url_route)
return _is_openai_compatible_host(parsed_url.hostname)
def is_gemini_route(
self, url_route: str, custom_llm_provider: Optional[str] = None
):

58
tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_openai_passthrough_logging_handler.py
View file

@ -257,6 +257,64 @@ class TestOpenAIPassthroughLoggingHandler:
)
assert OpenAIPassthroughLoggingHandler.is_openai_responses_route("") == False
def test_is_openai_route_recognizes_cognitiveservices_azure_com(self):
"""Azure OpenAI resources created via the newer "Azure AI Foundry" /
Cognitive Services pathway live on `*.cognitiveservices.azure.com`
subdomains rather than the older `openai.azure.com`. All four
is_openai_*_route methods must recognize both Azure subdomains so
cost tracking applies regardless of which Azure naming the user's
resource happens to be on.
"""
cognitive_chat = (
"https://my-resource.cognitiveservices.azure.com/v1/chat/completions"
)
cognitive_images_gen = (
"https://my-resource.cognitiveservices.azure.com/v1/images/generations"
)
cognitive_images_edit = (
"https://my-resource.cognitiveservices.azure.com/v1/images/edits"
)
cognitive_responses = (
"https://my-resource.cognitiveservices.azure.com/v1/responses"
)
assert (
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(
cognitive_chat
)
is True
)
assert (
OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(
cognitive_images_gen
)
is True
)
assert (
OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(
cognitive_images_edit
)
is True
)
assert (
OpenAIPassthroughLoggingHandler.is_openai_responses_route(
cognitive_responses
)
is True
)
# Cross-route negatives still hold for cognitiveservices hosts.
assert (
OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(
cognitive_responses
)
is False
)
assert (
OpenAIPassthroughLoggingHandler.is_openai_responses_route(cognitive_chat)
is False
)
@patch("litellm.completion_cost")
@patch(
"litellm.litellm_core_utils.litellm_logging.get_standard_logging_object_payload"