scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_ocr_rust_default
Some checks failed
LiteLLM Rust / rustfmt, clippy, test (push) Has been cancelled
Details

Browse source
This commit is contained in:
Devin AI 2026-07-29 21:38:37 +00:00
commit 5a095a7fb0
5 changed files with 106 additions and 156 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
litellm/model_prices_and_context_window_backup.json
View file

@ -13567,6 +13567,56 @@
}
]
},
"dashscope/qwen3.7-max": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "dashscope",
"max_input_tokens": 991808,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"dashscope/qwen3.7-plus": {
"litellm_provider": "dashscope",
"max_input_tokens": 991808,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tiered_pricing": [
{
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"range": [
0,
256000.0
]
},
{
"cache_read_input_token_cost": 2.4e-07,
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 4.8e-06,
"range": [
256000.0,
1000000.0
]
}
]
},
"dashscope/qwq-plus": {
"input_cost_per_token": 8e-07,
"litellm_provider": "dashscope",

50
model_prices_and_context_window.json
View file

@ -13567,6 +13567,56 @@
}
]
},
"dashscope/qwen3.7-max": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "dashscope",
"max_input_tokens": 991808,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true
},
"dashscope/qwen3.7-plus": {
"litellm_provider": "dashscope",
"max_input_tokens": 991808,
"max_output_tokens": 65536,
"max_tokens": 65536,
"mode": "chat",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"tiered_pricing": [
{
"cache_read_input_token_cost": 8e-08,
"input_cost_per_token": 4e-07,
"output_cost_per_token": 1.6e-06,
"range": [
0,
256000.0
]
},
{
"cache_read_input_token_cost": 2.4e-07,
"input_cost_per_token": 1.2e-06,
"output_cost_per_token": 4.8e-06,
"range": [
256000.0,
1000000.0
]
}
]
},
"dashscope/qwq-plus": {
"input_cost_per_token": 8e-07,
"litellm_provider": "dashscope",

11
tests/e2e/guardrails/guardrails_client.py
View file

@ -63,16 +63,6 @@ class OpenAIModerationParamsBody(GuardrailParamsBase):
model: str | None = None
class PresidioParamsBody(GuardrailParamsBase):
guardrail: Literal["presidio"] = "presidio"
presidio_analyzer_api_base: str | None = None
presidio_anonymizer_api_base: str | None = None
# apply_to_output masks PII the model itself emitted, which also makes the
# guardrail run post_call. logging_only masks what the proxy logs.
apply_to_output: bool | None = None
logging_only: bool | None = None
class BlockCodeExecutionParamsBody(GuardrailParamsBase):
guardrail: Literal["block_code_execution"] = "block_code_execution"
@ -81,7 +71,6 @@ GuardrailParamsBody = (
ContentFilterParamsBody
| BedrockGuardrailParamsBody
| OpenAIModerationParamsBody
| PresidioParamsBody
| BlockCodeExecutionParamsBody
)

141
tests/e2e/guardrails/test_presidio_guardrail_e2e.py
View file

@ -1,141 +0,0 @@
"""Live e2e: the built-in Presidio PII guardrail masks PII on the request and on
the model output.
Presidio replaces detected PII with `<ENTITY_TYPE>` placeholders (e.g.
`<EMAIL_ADDRESS>`) via a real analyzer + anonymizer. Two modes are checked
independently, each opted into per request (default_on=False) so it never touches
unrelated traffic:
- pre_call: the prompt is anonymized before it reaches the model, so a
repeat-verbatim request comes back with the placeholder, never the raw email
- post_call (apply_to_output): PII the model itself emits is masked on the way
out, so the caller never receives the raw value the model produced
A third mode, logging_only, is not covered here: the raw email stayed in the OTEL
span's `gen_ai.input.messages` on every attempt over a full poll deadline while
these two modes masked correctly, so that cell is tracked in LIT-4841 rather than
asserted against known-failing behavior.
Analyzer/anonymizer bases come from PRESIDIO_ANALYZER_API_BASE /
PRESIDIO_ANONYMIZER_API_BASE (compose provides the in-network hosts; point them at
locally published container ports for a host run). The chat backend is a gemini
deployment created for the test.
"""
from __future__ import annotations
import os
import time
from collections.abc import Callable
import pytest
from e2e_config import POLL_INTERVAL, POLL_TIMEOUT, unique_marker
from e2e_http import unwrap
from guardrails_client import GuardrailMode, GuardrailsClient, PresidioParamsBody
from lifecycle import ResourceManager
from models import ChatResponse
pytestmark = pytest.mark.e2e
RAW_EMAIL = "alice.example.person@example.com"
PLACEHOLDER = "<EMAIL_ADDRESS>"
ECHO_REQUEST = f"Repeat the following text back exactly, verbatim, with no changes: My email is {RAW_EMAIL}"
EMIT_REQUEST = f"Output exactly this one line and nothing else: Please contact {RAW_EMAIL} today"
def _content(response: ChatResponse) -> str:
if not response.choices:
return ""
message = response.choices[0].message
return (message.content if message else None) or ""
def _presidio_params(
mode: GuardrailMode, *, apply_to_output: bool = False, logging_only: bool = False
) -> PresidioParamsBody:
analyzer = os.environ["PRESIDIO_ANALYZER_API_BASE"]
anonymizer = os.environ["PRESIDIO_ANONYMIZER_API_BASE"]
return PresidioParamsBody(
mode=mode,
default_on=False,
presidio_analyzer_api_base=analyzer,
presidio_anonymizer_api_base=anonymizer,
apply_to_output=apply_to_output,
logging_only=logging_only,
)
def _poll_until_masked(call: Callable[[], str]) -> str:
"""Retry a call until the guardrail masks its PII, returning the last content.
Registering a guardrail is a control-plane write; the data-plane worker that
serves /chat/completions only picks it up on its next periodic DB sync (~30s
in proxy_server.py), so a call issued the instant after the create runs
against a worker that has no guardrail yet and passes the raw value through.
That is in-flight propagation, not a masking failure. Polling to the deadline
waits it out, so the assertions that follow judge the synced state; if the
mask never lands the last unmasked content is returned and they still fail.
"""
deadline = time.monotonic() + POLL_TIMEOUT
last = call()
while time.monotonic() < deadline:
if PLACEHOLDER in last and RAW_EMAIL not in last:
return last
time.sleep(POLL_INTERVAL)
last = call()
return last
class TestPresidioGuardrail:
@pytest.mark.covers(
"guardrail.presidio.pre_call.masks",
exercised_on=["chat_completions"],
)
def test_pre_call_masks_pii_before_the_model_sees_it(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
model = client.create_backend_model(resources, prefix="e2e-presidio-pre")
name = f"e2e-presidio-pre-{unique_marker()}"
guardrail_id = client.register(name, _presidio_params("pre_call"))
resources.defer(lambda: client.delete_guardrail(guardrail_id))
echoed = _poll_until_masked(
lambda: _content(
unwrap(client.chat(scoped_key, model, ECHO_REQUEST, guardrails=[name], max_tokens=128))
)
)
assert RAW_EMAIL not in echoed, (
"pre_call masking must strip the raw email before the model sees it, but the "
f"model echoed it back: {echoed[:300]!r}"
)
assert PLACEHOLDER in echoed, (
"the model should have echoed the masked placeholder the guardrail substituted, "
f"got: {echoed[:300]!r}"
)
@pytest.mark.covers(
"guardrail.presidio.post_call.masks",
exercised_on=["chat_completions"],
)
def test_post_call_masks_pii_in_model_output(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
model = client.create_backend_model(resources, prefix="e2e-presidio-post")
name = f"e2e-presidio-post-{unique_marker()}"
guardrail_id = client.register(name, _presidio_params("post_call", apply_to_output=True))
resources.defer(lambda: client.delete_guardrail(guardrail_id))
out = _poll_until_masked(
lambda: _content(
unwrap(client.chat(scoped_key, model, EMIT_REQUEST, guardrails=[name], max_tokens=128))
)
)
assert RAW_EMAIL not in out, (
"post_call masking must strip PII the model emitted, but the raw email reached the "
f"caller: {out[:300]!r}"
)
assert PLACEHOLDER in out, (
f"the masked placeholder should replace the model's PII output, got: {out[:300]!r}"
)

10
tests/test_litellm/test_cost_calculator.py
View file

@ -1014,9 +1014,9 @@ def test_tiered_pricing_only_deployment_selects_router_model_id():
router = Router(
model_list=[
{
"model_name": "qwen-3.7-plus",
"model_name": "qwen-tier-only",
"litellm_params": {
"model": "dashscope/qwen3.7-plus",
"model": "dashscope/qwen-tier-only-test",
"api_key": "sk-fake",
},
"model_info": {
@ -1037,10 +1037,12 @@ def test_tiered_pricing_only_deployment_selects_router_model_id():
assert entry.get("input_cost_per_token") is None
assert entry.get("tiered_pricing") is not None
# The stripped shared alias must not carry tiered pricing.
assert litellm.model_cost["dashscope/qwen3.7-plus"].get("tiered_pricing") is None
assert (
litellm.model_cost["dashscope/qwen-tier-only-test"].get("tiered_pricing") is None
)
selected = _select_model_name_for_cost_calc(
model="dashscope/qwen3.7-plus",
model="dashscope/qwen-tier-only-test",
completion_response=None,
custom_pricing=True,
custom_llm_provider="dashscope",