mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'upstream/main' into fix/presidio-session-lock-concurrency
This commit is contained in:
commit
e35b9206fe
970 changed files with 26867 additions and 10668 deletions
9
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
9
.github/PULL_REQUEST_TEMPLATE/general.md
vendored
|
|
@ -4,6 +4,13 @@
|
|||
Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
|
||||
"## Affected release" with nothing under it must not appear in the final description -->
|
||||
|
||||
<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
|
||||
Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
|
||||
No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
|
||||
|
||||
<details>
|
||||
<summary>Details</summary>
|
||||
|
||||
## TLDR
|
||||
|
||||
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
|
||||
|
|
@ -164,3 +171,5 @@ Example checklists:
|
|||
## Final Attestation
|
||||
|
||||
- [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
|
||||
|
||||
</details>
|
||||
|
|
|
|||
7
.github/workflows/test-unit.yml
vendored
7
.github/workflows/test-unit.yml
vendored
|
|
@ -433,6 +433,13 @@ jobs:
|
|||
|
||||
- shard: rust-bridge-harness
|
||||
test-path: >-
|
||||
tests/test_litellm_rust/test_dispatch.py
|
||||
tests/test_litellm_rust/test_public_entries.py
|
||||
tests/test_litellm_rust/chat_completions/test_dispatch.py
|
||||
tests/test_litellm_rust/embeddings/test_dispatch.py
|
||||
tests/test_litellm_rust/messages/test_dispatch.py
|
||||
tests/test_litellm_rust/ocr/test_dispatch.py
|
||||
tests/test_litellm_rust/responses/test_dispatch.py
|
||||
tests/unit/compression
|
||||
tests/unit/harness
|
||||
tests/unit/rust_bridge
|
||||
|
|
|
|||
41
cookbook/litellm_proxy_server/batch_api/bedrock/README.md
Normal file
41
cookbook/litellm_proxy_server/batch_api/bedrock/README.md
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
# Bedrock batch endpoints
|
||||
|
||||
Set `aws_bedrock_control_plane_endpoint` on a deployment when batch create, retrieve, and cancel requests must use a custom Bedrock control-plane endpoint. The proxy signs these requests for the configured URL. Without this setting, batch requests use the standard regional Bedrock endpoint
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: bedrock-batch
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
|
||||
aws_region_name: us-east-1
|
||||
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
|
||||
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
|
||||
aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
|
||||
s3_output_bucket_name: os.environ/AWS_S3_OUTPUT_BUCKET_NAME
|
||||
aws_bedrock_control_plane_endpoint: os.environ/BEDROCK_CONTROL_PLANE_ENDPOINT
|
||||
```
|
||||
|
||||
Set `BEDROCK_CONTROL_PLANE_ENDPOINT` to your Bedrock control-plane VPC endpoint or compatible gateway base URL, without `/model-invocation-job`. The proxy adds the operation path. The endpoint must be reachable from the proxy, and the configured AWS region must match the endpoint's signing region
|
||||
|
||||
`api_base` and `aws_bedrock_runtime_endpoint` configure runtime inference separately. They do not override the batch control-plane endpoint. `s3_endpoint_url` configures S3 file access separately
|
||||
|
||||
The control-plane setting belongs in deployment configuration. Sending `aws_bedrock_control_plane_endpoint` in an API request body is rejected. There is no automatic `AWS_BEDROCK_CONTROL_PLANE_ENDPOINT` environment fallback, so reference the desired environment variable explicitly as shown above
|
||||
|
||||
Start the proxy with this configuration and set `LITELLM_API_BASE` to its URL. Set `BEDROCK_BATCH_INPUT_URI` to a valid input JSONL file in S3 that the batch service role can read
|
||||
|
||||
```bash
|
||||
curl -sS "$LITELLM_API_BASE/v1/batches" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"model\":\"bedrock-batch\",\"input_file_id\":\"$BEDROCK_BATCH_INPUT_URI\",\"endpoint\":\"/v1/chat/completions\",\"completion_window\":\"24h\"}"
|
||||
```
|
||||
|
||||
Use the returned `id` as `BATCH_ID` to retrieve or cancel the same job through the deployment's configured endpoint
|
||||
|
||||
```bash
|
||||
curl -sS "$LITELLM_API_BASE/v1/batches/$BATCH_ID?model=bedrock-batch" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
|
||||
|
||||
curl -sS -X POST "$LITELLM_API_BASE/v1/batches/$BATCH_ID/cancel?model=bedrock-batch" \
|
||||
-H "Authorization: Bearer $LITELLM_MASTER_KEY"
|
||||
```
|
||||
|
|
@ -852,20 +852,26 @@ class CheckBatchCost:
|
|||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
await logging_obj.async_success_handler(
|
||||
result=response,
|
||||
batch_cost=batch_result.cost,
|
||||
batch_usage=batch_result.usage,
|
||||
batch_models=batch_result.models,
|
||||
batch_successful_requests=batch_result.successful_requests,
|
||||
batch_failed_requests=batch_result.failed_requests,
|
||||
batch_prompt_cost=batch_result.prompt_cost,
|
||||
batch_completion_cost=batch_result.completion_cost,
|
||||
)
|
||||
except Exception:
|
||||
await self._release_job_claim(job, original_status, original_file_object)
|
||||
raise
|
||||
from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
|
||||
|
||||
with track_batch_accounting(logging_obj.litellm_call_id) as accounting:
|
||||
try:
|
||||
await logging_obj.async_success_handler(
|
||||
result=response,
|
||||
batch_cost=batch_result.cost,
|
||||
batch_usage=batch_result.usage,
|
||||
batch_models=batch_result.models,
|
||||
batch_successful_requests=batch_result.successful_requests,
|
||||
batch_failed_requests=batch_result.failed_requests,
|
||||
batch_prompt_cost=batch_result.prompt_cost,
|
||||
batch_completion_cost=batch_result.completion_cost,
|
||||
)
|
||||
finally:
|
||||
if accounting.retryable_failure:
|
||||
await self._release_job_claim(job, original_status, original_file_object)
|
||||
|
||||
if accounting.retryable_failure:
|
||||
return None
|
||||
|
||||
# Record batch duration (completed_at - created_at)
|
||||
if prom_logger and response.completed_at and response.created_at:
|
||||
|
|
@ -1186,7 +1192,7 @@ class CheckBatchCost:
|
|||
continue
|
||||
verbose_proxy_logger.error(
|
||||
f"CheckBatchCost: failed to track cost for batch {batch_id} "
|
||||
f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
|
||||
f"(job {job.id}): {tracking_err}"
|
||||
)
|
||||
self._record_error(prom_logger, "cost_tracking_error")
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.78"
|
||||
version = "0.1.79"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.78"
|
||||
version = "0.1.79"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.110"
|
||||
version = "0.4.111"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.110"
|
||||
version = "0.4.111"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ mod selection;
|
|||
|
||||
pub(crate) use native::NativeCacheHandle;
|
||||
pub(crate) use python::{CacheCall, PythonCache};
|
||||
pub(crate) use selection::{Cached, Selection, admit_native, configure, configured_native};
|
||||
pub(crate) use selection::{Cached, Selection, configure, configured_native};
|
||||
|
||||
use litellm_cache::Error;
|
||||
use pyo3::{
|
||||
|
|
|
|||
|
|
@ -279,7 +279,7 @@ pub(in crate::cache) fn configured(
|
|||
)> {
|
||||
let handle = native_handle(configured)?.ok_or_else(|| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err(
|
||||
"the configured cache changed to a Python cache after native admission",
|
||||
"native inference needs a v2 cache, the catalog keeps Python caches on Python",
|
||||
)
|
||||
})?;
|
||||
let cache = handle.extract::<PyRef<'_, NativeCacheHandle>>()?;
|
||||
|
|
|
|||
|
|
@ -76,21 +76,6 @@ fn selected_cache<'py>(
|
|||
Ok(Some(configured))
|
||||
}
|
||||
|
||||
pub(crate) fn admit_native(
|
||||
py: Python<'_>,
|
||||
kwargs: &Bound<'_, PyDict>,
|
||||
call_type: &str,
|
||||
) -> PyResult<()> {
|
||||
if let Some(configured) = selected_cache(py, kwargs, call_type)?
|
||||
&& native::v2::native_handle(&configured)?.is_none()
|
||||
{
|
||||
return Err(crate::errors::RustBridgeDeclined::new_err(
|
||||
"the configured cache requires Python inference",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn configured_native(
|
||||
py: Python<'_>,
|
||||
kwargs: &Bound<'_, PyDict>,
|
||||
|
|
|
|||
|
|
@ -177,7 +177,6 @@ where
|
|||
} else {
|
||||
R::SYNC_CALL_TYPE
|
||||
};
|
||||
crate::cache::admit_native(py, &call.kwargs, call_type)?;
|
||||
let (arguments, hooks) = super::call_hooks(py, R::OPERATION, &call, asynchronous)?;
|
||||
super::run_public_call(
|
||||
py,
|
||||
|
|
|
|||
|
|
@ -8,59 +8,19 @@ use litellm_auth::AuthServices;
|
|||
use litellm_cache_response::{CachePolicy, ScopedCache};
|
||||
use litellm_callbacks_legacy_python::LoggingOperation;
|
||||
use litellm_host::{call::HostedMachine, protocol::Protocol};
|
||||
use litellm_host_python::present;
|
||||
use litellm_inference_responses::{ResponsesRoute, route::Responses};
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::Value;
|
||||
pub(crate) use websocket::ResponsesWebSocketConnection;
|
||||
|
||||
use super::{
|
||||
NativeCall,
|
||||
inference::{InferenceHost, InferenceRoute, run_inference},
|
||||
};
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.responses";
|
||||
|
||||
fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult<Py<PyAny>> {
|
||||
let resolved = call.resolved()?;
|
||||
if let Some(reason) = py
|
||||
.import(ROUTE_HOST_MODULE)?
|
||||
.getattr("decline_reason")?
|
||||
.call1((&resolved,))?
|
||||
.extract::<Option<String>>()?
|
||||
{
|
||||
return Err(RustBridgeDeclined::new_err(reason));
|
||||
}
|
||||
let argument = |name: &str| present(&call.kwargs, &call.base, name);
|
||||
let model = argument("model")?
|
||||
.ok_or_else(|| pyo3::exceptions::PyValueError::new_err("model is required"))?
|
||||
.extract::<String>()?;
|
||||
let provider = argument("custom_llm_provider")?
|
||||
.map(|value| value.extract::<String>())
|
||||
.transpose()?;
|
||||
if provider
|
||||
.as_deref()
|
||||
.is_some_and(|provider| provider != "openai")
|
||||
|| model
|
||||
.strip_prefix("openai/")
|
||||
.unwrap_or(&model)
|
||||
.contains('/')
|
||||
{
|
||||
return Err(RustBridgeDeclined::new_err(
|
||||
"native HTTP responses provider",
|
||||
));
|
||||
}
|
||||
if argument("stream")?
|
||||
.map(|value| litellm_host_python::from_py::<Value>(&value))
|
||||
.transpose()?
|
||||
.is_some_and(|value| value == Value::Bool(true))
|
||||
{
|
||||
return Err(RustBridgeDeclined::new_err(
|
||||
"native Python responses streaming",
|
||||
));
|
||||
}
|
||||
let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
|
||||
run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -551,6 +551,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
|
|||
|
||||
model_cost = get_model_cost_map(url=model_cost_map_url)
|
||||
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
|
||||
apply_search_free_tier: bool = False
|
||||
cost_margin_config: Dict[
|
||||
str, Union[float, Dict[str, float]]
|
||||
] = {} # Provider-specific or global cost margins. Examples:
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ class A2ACompletionBridgeHandler:
|
|||
@staticmethod
|
||||
def _build_completion_params(
|
||||
params: dict[str, Any],
|
||||
litellm_params: Mapping[str, Any],
|
||||
litellm_params: Mapping[str, object],
|
||||
api_base: str | None,
|
||||
agent_extra_headers: Mapping[str, str] | None,
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -59,16 +59,6 @@ class PendingBatchRead:
|
|||
previous_access_times: dict[str, float | None]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeclaredBatchRead:
|
||||
"""A ``async_batch_get_cache`` split in two: the memory half done, the Redis half declared on a ``RedisBatch``
|
||||
so it rides that batch's next round trip, resolved later with ``async_resolve_batch_get``."""
|
||||
|
||||
keys: tuple[str, ...]
|
||||
pending: PendingBatchRead
|
||||
result: BatchResult[Mapping[str, object]] | None
|
||||
|
||||
|
||||
def _log_deferred_increment_failure(future: asyncio.Future[float]) -> None:
|
||||
if future.cancelled():
|
||||
return
|
||||
|
|
@ -382,22 +372,6 @@ class DualCache(BaseCache):
|
|||
await self.in_memory_cache.async_set_cache(key, value, **self._backfill_kwargs(kwargs))
|
||||
return merged
|
||||
|
||||
async def declare_batch_get(self, keys: Sequence[str], batch: RedisBatch) -> DeclaredBatchRead:
|
||||
pending: Final = await self._prepare_batch_get(
|
||||
list(keys),
|
||||
local_only=False,
|
||||
throttle_redis=False,
|
||||
)
|
||||
return DeclaredBatchRead(
|
||||
keys=tuple(keys),
|
||||
pending=pending,
|
||||
result=batch.mget(pending.redis_keys) if pending.redis_keys else None,
|
||||
)
|
||||
|
||||
async def async_resolve_batch_get(self, declared: DeclaredBatchRead) -> list[object | None]:
|
||||
redis_result: Final = None if declared.result is None else await declared.result
|
||||
return await self._apply_batch_get(declared.pending, redis_result)
|
||||
|
||||
async def async_batch_get_cache(
|
||||
self,
|
||||
keys: list,
|
||||
|
|
|
|||
|
|
@ -134,7 +134,12 @@ _INCREMENT_WITH_FLOOR_LUA: Final = (
|
|||
"return count"
|
||||
)
|
||||
|
||||
_INCREMENT_IF_PRESENT_LUA: Final = (
|
||||
"if redis.call('EXISTS', KEYS[1]) == 0 then return false end return redis.call('INCRBYFLOAT', KEYS[1], ARGV[1])"
|
||||
)
|
||||
|
||||
_LUA_COUNT: Final = TypeAdapter(int)
|
||||
_LUA_OPTIONAL_TOTAL: Final[TypeAdapter[float | None]] = TypeAdapter(float | None)
|
||||
_OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
|
||||
|
||||
|
||||
|
|
@ -675,7 +680,7 @@ class RedisCache(BaseCache):
|
|||
# redis namespaces
|
||||
self.namespace = namespace
|
||||
# for high traffic, we store the redis results in memory and then batch write to redis
|
||||
self.redis_batch_writing_buffer: list = []
|
||||
self.redis_batch_writing_buffer: list[tuple[str, object]] = []
|
||||
if redis_flush_size is None:
|
||||
self.redis_flush_size: int = 100
|
||||
else:
|
||||
|
|
@ -1568,6 +1573,46 @@ class RedisCache(BaseCache):
|
|||
count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
|
||||
return _LUA_COUNT.validate_python(count)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_increment_if_present(
|
||||
self, key: str, value: float, parent_otel_span: Span | None = None
|
||||
) -> float | None:
|
||||
_redis_client: Final = self._async_commands()
|
||||
namespaced_key: Final = self.check_and_fix_namespace(key=key)
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
total: Final = _LUA_OPTIONAL_TOTAL.validate_python(
|
||||
await _redis_client.eval(_INCREMENT_IF_PRESENT_LUA, 1, namespaced_key, value)
|
||||
)
|
||||
except Exception as e:
|
||||
failed_at: Final = time.time()
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_failure_hook(
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=failed_at - start_time,
|
||||
error=e,
|
||||
call_type="async_increment_if_present",
|
||||
caller=_get_call_stack_info(),
|
||||
start_time=start_time,
|
||||
end_time=failed_at,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
)
|
||||
raise
|
||||
end_time: Final = time.time()
|
||||
asyncio.create_task(
|
||||
self.service_logger_obj.async_service_success_hook(
|
||||
service=ServiceTypes.REDIS,
|
||||
duration=end_time - start_time,
|
||||
call_type="async_increment_if_present",
|
||||
caller=_get_call_stack_info(),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
)
|
||||
return total
|
||||
|
||||
async def flush_cache_buffer(self):
|
||||
print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
|
||||
await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,19 @@
|
|||
import inspect
|
||||
from collections.abc import Awaitable, Callable, Coroutine, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
|
||||
|
||||
from litellm import main
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext
|
||||
from litellm.rust_bridge import detectors
|
||||
from litellm.rust_bridge.catalog import Feature, Route, RouteContext
|
||||
from litellm.rust_bridge.chat_completions import detectors as route_detectors
|
||||
from litellm.rust_bridge.chat_completions.entrypoints import (
|
||||
NATIVE_ACOMPLETION,
|
||||
NATIVE_COMPLETION,
|
||||
)
|
||||
from litellm.rust_bridge.dispatch import PublicDispatch
|
||||
from litellm.rust_bridge.public_call import (
|
||||
AsyncFlag,
|
||||
NativeCall,
|
||||
bind,
|
||||
native_call,
|
||||
|
|
@ -44,6 +48,7 @@ def _python_acompletion() -> PythonAcompletion:
|
|||
|
||||
_PYTHON_COMPLETION: Final = _python_completion()
|
||||
_COMPLETION: Final = signature(_PYTHON_COMPLETION)
|
||||
_FLAG: Final = AsyncFlag("acompletion", sync=_COMPLETION)
|
||||
_PYTHON_ACOMPLETION: Final = _python_acompletion()
|
||||
_ACOMPLETION: Final = signature(_PYTHON_ACOMPLETION)
|
||||
|
||||
|
|
@ -61,11 +66,32 @@ def _public_request(
|
|||
return native_call(legacy, args, kwargs)
|
||||
|
||||
|
||||
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
||||
{
|
||||
**detectors.COMMON,
|
||||
Feature.MOCK_RESPONSE: route_detectors.chat_mock_response,
|
||||
Feature.AGENTIC_LOOP_HOOK: route_detectors.chat_agentic_loop_hook,
|
||||
Feature.RESPONSE_CACHE: detectors.response_cache,
|
||||
Feature.STREAMING: detectors.streaming,
|
||||
Feature.DROP_PARAMS: detectors.drop_params,
|
||||
Feature.MODIFY_PARAMS: detectors.modify_params,
|
||||
Feature.EXTRA_BODY: detectors.extra_body,
|
||||
Feature.RETRIES_FALLBACKS: route_detectors.chat_retries_fallbacks,
|
||||
Feature.PROMPT_MANAGEMENT: detectors.prompt_management,
|
||||
Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection,
|
||||
Feature.MODEL_PARAM_GATING: route_detectors.chat_model_param_gating,
|
||||
Feature.MODEL_LIST: route_detectors.model_list,
|
||||
Feature.MODEL_ALIAS_MAP: route_detectors.model_alias_map,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _context(request: NativeCall) -> RouteContext:
|
||||
return RouteContext(
|
||||
Route.CHAT_COMPLETIONS,
|
||||
provider=optional_str(request.resolved.get("custom_llm_provider")),
|
||||
model=str(request.resolved["model"]),
|
||||
features=detectors.detected(_FEATURES, request),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -73,7 +99,6 @@ _DISPATCH: Final = PublicDispatch(
|
|||
route=Route.CHAT_COMPLETIONS,
|
||||
request=lambda args, kwargs: _public_request(_COMPLETION, args, kwargs),
|
||||
context=_context,
|
||||
bypass=lambda request: request.kwargs.get("acompletion") is True,
|
||||
)
|
||||
|
||||
_ADISPATCH: Final = PublicDispatch(
|
||||
|
|
@ -88,9 +113,11 @@ def completion(
|
|||
**kwargs: object, # kwargs-ok: preserve the public chat completions call shape
|
||||
) -> ChatResult | Coroutine[object, object, ChatResult]:
|
||||
python: Final = _PYTHON_COMPLETION
|
||||
if _FLAG.requested(kwargs):
|
||||
return acompletion(**_FLAG.keywords(args, kwargs))
|
||||
return _DISPATCH.run(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=python,
|
||||
binding=NATIVE_COMPLETION,
|
||||
native=native_call_hook,
|
||||
|
|
@ -101,7 +128,7 @@ async def acompletion(*args: object, **kwargs: object) -> ChatResult: # kwargs-
|
|||
python: Final = _PYTHON_ACOMPLETION
|
||||
return await _ADISPATCH.arun(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=python,
|
||||
binding=NATIVE_ACOMPLETION,
|
||||
native=native_call_hook,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ CLAUDE_CODE_CLIENT: Final = "claude-code"
|
|||
MICROSOFT_365_COPILOT_DEFAULT_TIME_ZONE: Final = "UTC"
|
||||
GITHUB_COPILOT_PER_USER_AUTH_TYPE: Final = "per_user_oauth"
|
||||
GITHUB_COPILOT_AUTH_TYPE_KEY: Final = "github_copilot_auth_type"
|
||||
GITHUB_COPILOT_HOST_KEY: Final = "github_copilot_host"
|
||||
GITHUB_COPILOT_DEFAULT_HOST: Final = "github.com"
|
||||
GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS: Final = 60
|
||||
GITHUB_COPILOT_USER_CREDENTIAL_CACHE_TTL_SECONDS: Final = 60
|
||||
GITHUB_COPILOT_DEVICE_FLOW_CACHE_PREFIX: Final = "github_copilot_device_flow"
|
||||
|
|
@ -1812,6 +1814,7 @@ LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, get_env_int("LITELLM_EXECUTED
|
|||
########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
|
||||
LITELLM_CLI_SOURCE_IDENTIFIER: Final = "litellm-cli"
|
||||
LITELLM_CLI_SESSION_TOKEN_PREFIX: Final = "litellm-session-token"
|
||||
CP_RETURN_TO_COOKIE_PREFIX: Final = "return_to.v1."
|
||||
CLI_SSO_SESSION_CACHE_KEY_PREFIX: Final = "cli_sso_session"
|
||||
CLI_SSO_SESSION_TTL_SECONDS: Final = 600
|
||||
CLI_SESSION_KEY_PREFIX: Final = "cli-session"
|
||||
|
|
@ -1855,6 +1858,9 @@ SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = get_env_float(
|
|||
SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", 300.0)
|
||||
SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", 30.0)
|
||||
SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = get_env_int("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", 100000)
|
||||
SPEND_LOG_SIZE_CHECK_JOB_NAME: Final = "spend_log_size_check"
|
||||
SPEND_LOG_SIZE_CHECK_INTERVAL: Final = os.getenv("SPEND_LOG_SIZE_CHECK_INTERVAL", "1d")
|
||||
SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES: Final = get_env_int("SPEND_LOG_SIZE_CLEANUP_RECHECK_BATCHES", 10)
|
||||
SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = get_env_float(
|
||||
"SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", 5.0
|
||||
)
|
||||
|
|
@ -1976,6 +1982,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
|
|||
"public_model_groups",
|
||||
"public_model_groups_links",
|
||||
"cost_discount_config",
|
||||
"apply_search_free_tier",
|
||||
"cost_margin_config",
|
||||
"block_requests_for_models_without_pricing",
|
||||
"budget_exceeded_throttle_percentage",
|
||||
|
|
@ -2316,6 +2323,7 @@ HARNESS_STDERR_TAIL_LINES: Final = 40
|
|||
HARNESS_STREAM_READ_CHUNK_BYTES: Final = 64 * 1024
|
||||
HARNESS_EVENT_QUEUE_MAX_SIZE: Final = 1024
|
||||
HARNESS_PROCESS_KILL_GRACE_SECONDS: Final = 5.0
|
||||
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS: Final = 30.0
|
||||
HARNESS_SNAPSHOT_SKIP_DIRS: Final = frozenset(
|
||||
{
|
||||
".git",
|
||||
|
|
|
|||
|
|
@ -906,11 +906,15 @@ def _bedrock_pricing_info(
|
|||
return model_info_with_pricing_overrides(profile, registration[1])
|
||||
|
||||
|
||||
def _declares_rates(model_info: ModelInfo | None) -> bool:
|
||||
entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
|
||||
def cost_map_key_prices_anything(key: str) -> bool:
|
||||
entry: Final = _raw_cost_map_entry(key)
|
||||
return entry is not None and _cost_map_entry_prices_anything(entry)
|
||||
|
||||
|
||||
def _declares_rates(model_info: ModelInfo | None) -> bool:
|
||||
return model_info is not None and cost_map_key_prices_anything(model_info["key"])
|
||||
|
||||
|
||||
def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | None) -> str | None:
|
||||
"""The provider to price an explicit base_model under, when the deployment's own provider has
|
||||
no rates for it (e.g. chatgpt/ subscription rows) but the base model's own row does"""
|
||||
|
|
@ -1507,6 +1511,7 @@ def completion_cost(
|
|||
litellm_model_name: str | None = None,
|
||||
router_model_id: str | None = None,
|
||||
litellm_logging_obj: LitellmLoggingObject | None = None,
|
||||
pricing_provider: str | None = None,
|
||||
### SERVICE TIER ###
|
||||
service_tier: str | None = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
|
|
@ -1527,6 +1532,8 @@ def completion_cost(
|
|||
"""
|
||||
try:
|
||||
call_type = infer_call_type(call_type, completion_response) or "completion"
|
||||
if pricing_provider is not None:
|
||||
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
|
||||
|
||||
if call_type == CallTypes.aresponses_websocket.value and isinstance(
|
||||
completion_response, LiteLLMRealtimeStreamLoggingObject
|
||||
|
|
@ -1552,6 +1559,7 @@ def completion_cost(
|
|||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
pricing_provider=pricing_provider,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
|
|
@ -1642,10 +1650,10 @@ def completion_cost(
|
|||
*(((model, request_pricing_provider),) if model is not None else ()),
|
||||
)
|
||||
potential_model_names: Final = (
|
||||
raw_candidates if explicit_pricing else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
)
|
||||
|
||||
for idx, (model, pricing_provider) in enumerate(potential_model_names):
|
||||
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
try:
|
||||
if verbose_logger.isEnabledFor(logging.DEBUG):
|
||||
verbose_logger.debug("selected model name for cost calculation: %s", model)
|
||||
|
|
@ -1712,7 +1720,9 @@ def completion_cost(
|
|||
|
||||
hidden_params = getattr(completion_response, "_hidden_params", None)
|
||||
if hidden_params is not None:
|
||||
custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None)
|
||||
custom_llm_provider = pricing_provider or hidden_params.get(
|
||||
"custom_llm_provider", custom_llm_provider or None
|
||||
)
|
||||
region_name = hidden_params.get("region_name", region_name)
|
||||
|
||||
else:
|
||||
|
|
@ -1998,7 +2008,7 @@ def completion_cost(
|
|||
prompt_tokens=prompt_tokens or 0,
|
||||
completion_tokens=completion_tokens or 0,
|
||||
custom_llm_provider=_pricing_provider_for_cost_candidate(
|
||||
candidate_provider=pricing_provider,
|
||||
candidate_provider=candidate_pricing_provider,
|
||||
request_provider=request_pricing_provider,
|
||||
current_provider=custom_llm_provider,
|
||||
),
|
||||
|
|
@ -2102,7 +2112,7 @@ def completion_cost(
|
|||
_token_type_breakdown = get_token_type_cost_breakdown(
|
||||
model=model,
|
||||
custom_llm_provider=_pricing_provider_for_cost_candidate(
|
||||
candidate_provider=pricing_provider,
|
||||
candidate_provider=candidate_pricing_provider,
|
||||
request_provider=request_pricing_provider,
|
||||
current_provider=custom_llm_provider,
|
||||
),
|
||||
|
|
@ -2225,6 +2235,7 @@ def response_cost_calculator(
|
|||
litellm_model_name: str | None = None,
|
||||
router_model_id: str | None = None,
|
||||
litellm_logging_obj: LitellmLoggingObject | None = None,
|
||||
pricing_provider: str | None = None,
|
||||
### SERVICE TIER ###
|
||||
service_tier: str | None = None, # for OpenAI service tier pricing
|
||||
### DATA RESIDENCY ###
|
||||
|
|
@ -2265,6 +2276,7 @@ def response_cost_calculator(
|
|||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
pricing_provider=pricing_provider,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from pydantic_core import to_jsonable_python
|
|||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm.cost_calculator import cost_map_key_prices_anything
|
||||
from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -21,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
|||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.decisions.transformation import (
|
||||
BaseDecisionsConfig,
|
||||
DecisionsResponseValidationError,
|
||||
image_media_type,
|
||||
image_param,
|
||||
input_images,
|
||||
|
|
@ -687,6 +689,13 @@ def _request_body(call: DecisionsCall) -> Mapping[str, object]:
|
|||
return body
|
||||
|
||||
|
||||
def _pricing_model(response: DecisionsIRResponse, call: DecisionsCall) -> str:
|
||||
returned: Final = get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
if returned is not None and cost_map_key_prices_anything(returned):
|
||||
return returned
|
||||
return f"{call.custom_llm_provider}/{call.model}"
|
||||
|
||||
|
||||
def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
formatted: Final = _formatted_response(response, call)
|
||||
provider_cost: Final = call.provider_config.provider_reported_cost(response)
|
||||
|
|
@ -694,10 +703,7 @@ def _format_response(response: DecisionsIRResponse, call: DecisionsCall) -> Deci
|
|||
{
|
||||
"model": f"{call.custom_llm_provider}/{call.model}",
|
||||
"custom_llm_provider": call.custom_llm_provider,
|
||||
"provider_response_model": (
|
||||
get_model_cost_key(f"{call.custom_llm_provider}/{response.model}")
|
||||
or f"{call.custom_llm_provider}/{call.model}"
|
||||
),
|
||||
"provider_response_model": _pricing_model(response, call),
|
||||
**({"additional_headers": {RESPONSE_COST_HEADER: provider_cost}} if provider_cost is not None else {}),
|
||||
}
|
||||
)
|
||||
|
|
@ -727,18 +733,21 @@ def _formatted_response(
|
|||
|
||||
|
||||
def _map_upstream_exception(error: Exception, call: DecisionsCall) -> Exception:
|
||||
if isinstance(error, BaseLLMException) and error.status_code_is_synthesized:
|
||||
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
|
||||
return litellm.APIConnectionError(
|
||||
message=f"{provider_label} - {error.message}",
|
||||
llm_provider=call.custom_llm_provider,
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
)
|
||||
return litellm.exception_type(
|
||||
model=f"{call.custom_llm_provider}/{call.model}",
|
||||
custom_llm_provider=call.custom_llm_provider,
|
||||
original_exception=error,
|
||||
)
|
||||
provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception"
|
||||
model: Final = f"{call.custom_llm_provider}/{call.model}"
|
||||
match error:
|
||||
case DecisionsResponseValidationError():
|
||||
return litellm.InternalServerError(
|
||||
message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
|
||||
)
|
||||
case BaseLLMException(status_code_is_synthesized=True):
|
||||
return litellm.APIConnectionError(
|
||||
message=f"{provider_label} - {error.message}", llm_provider=call.custom_llm_provider, model=model
|
||||
)
|
||||
case _:
|
||||
return litellm.exception_type(
|
||||
model=model, custom_llm_provider=call.custom_llm_provider, original_exception=error
|
||||
)
|
||||
|
||||
|
||||
async def asend(call: DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse:
|
||||
|
|
|
|||
|
|
@ -9,7 +9,15 @@ from litellm.rust_bridge.embeddings.entrypoints import (
|
|||
NATIVE_AEMBEDDING,
|
||||
NATIVE_EMBEDDING,
|
||||
)
|
||||
from litellm.rust_bridge.public_call import NativeCall, bind, native_call, native_call_hook, optional_str, signature
|
||||
from litellm.rust_bridge.public_call import (
|
||||
AsyncFlag,
|
||||
NativeCall,
|
||||
bind,
|
||||
native_call,
|
||||
native_call_hook,
|
||||
optional_str,
|
||||
signature,
|
||||
)
|
||||
from litellm.types.utils import EmbeddingResponse
|
||||
|
||||
__all__ = ("aembedding", "embedding")
|
||||
|
|
@ -24,6 +32,7 @@ _PYTHON_AEMBEDDING: Final = cast( # cast-ok: [LIT006] preserve the legacy publi
|
|||
PythonAembedding, main.aembedding
|
||||
)
|
||||
_EMBEDDING_SIGNATURE: Final = signature(_PYTHON_EMBEDDING)
|
||||
_FLAG: Final = AsyncFlag("aembedding", sync=_EMBEDDING_SIGNATURE)
|
||||
|
||||
|
||||
def _public_request(
|
||||
|
|
@ -50,7 +59,6 @@ _DISPATCH: Final = PublicDispatch(
|
|||
route=Route.EMBEDDINGS,
|
||||
request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs),
|
||||
context=_context,
|
||||
bypass=lambda request: request.kwargs.get("aembedding") is True,
|
||||
)
|
||||
|
||||
_ADISPATCH: Final = PublicDispatch(
|
||||
|
|
@ -64,9 +72,11 @@ def embedding(
|
|||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: preserve the public embedding call shape
|
||||
) -> EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]:
|
||||
if _FLAG.requested(kwargs):
|
||||
return aembedding(**_FLAG.keywords(args, kwargs))
|
||||
return _DISPATCH.run(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=_PYTHON_EMBEDDING,
|
||||
binding=NATIVE_EMBEDDING,
|
||||
native=native_call_hook,
|
||||
|
|
@ -76,7 +86,7 @@ def embedding(
|
|||
async def aembedding(*args: object, **kwargs: object) -> EmbeddingResponse: # kwargs-ok: preserve the public call shape
|
||||
return await _ADISPATCH.arun(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=_PYTHON_AEMBEDDING,
|
||||
binding=NATIVE_AEMBEDDING,
|
||||
native=native_call_hook,
|
||||
|
|
|
|||
|
|
@ -292,6 +292,7 @@ class _MCPResponseStream(httpx2.AsyncByteStream):
|
|||
class _MCPHTTPClient(httpx2.AsyncClient):
|
||||
cleanup_scope: anyio.CancelScope | None = None
|
||||
cleanup_errors: tuple[Exception, ...] = ()
|
||||
post_failure: asyncio.Future[Exception] | None = None
|
||||
|
||||
def _record_cleanup_error(self, error: Exception) -> None:
|
||||
if self.cleanup_scope is not None and self.cleanup_scope.shield:
|
||||
|
|
@ -319,7 +320,12 @@ class _MCPHTTPClient(httpx2.AsyncClient):
|
|||
response: Final = await super().send(request, stream=stream, auth=auth, follow_redirects=follow_redirects)
|
||||
if request.method == "POST" and response.is_error and response.status_code != 404:
|
||||
await response.aclose()
|
||||
response.raise_for_status()
|
||||
try:
|
||||
response.raise_for_status()
|
||||
except httpx2.HTTPStatusError as error:
|
||||
if self.post_failure is not None and not self.post_failure.done():
|
||||
self.post_failure.set_result(error)
|
||||
raise
|
||||
if stream:
|
||||
response.stream = _MCPResponseStream(
|
||||
cast(httpx2.AsyncByteStream, response.stream), self._record_cleanup_error
|
||||
|
|
@ -757,6 +763,8 @@ class MCPClient:
|
|||
read_stream: Final = transport[0]
|
||||
write_stream: Final = transport[1]
|
||||
stream_error: Final[asyncio.Future[Exception]] = asyncio.get_running_loop().create_future()
|
||||
if isinstance(http_client, _MCPHTTPClient):
|
||||
http_client.post_failure = stream_error # rebind-ok: operation-scoped stream error
|
||||
_notify_stream_error(on_stream_error, stream_error)
|
||||
|
||||
session_kwargs: Final = (
|
||||
|
|
@ -823,6 +831,7 @@ class MCPClient:
|
|||
if isinstance(http_client, _MCPHTTPClient):
|
||||
http_client.cleanup_errors = ()
|
||||
http_client.cleanup_scope = None
|
||||
http_client.post_failure = None # rebind-ok: clear operation-scoped stream error
|
||||
await anyio.lowlevel.checkpoint_if_cancelled()
|
||||
if cleanup_scope.cancel_called:
|
||||
raise (
|
||||
|
|
|
|||
|
|
@ -12,17 +12,27 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import os
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import HARNESS_STDERR_TAIL_LINES, HARNESS_STREAM_READ_CHUNK_BYTES
|
||||
from litellm.constants import (
|
||||
HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS,
|
||||
HARNESS_STDERR_TAIL_LINES,
|
||||
HARNESS_STREAM_READ_CHUNK_BYTES,
|
||||
)
|
||||
from litellm.harness.context import SessionContext
|
||||
from litellm.harness.errors import HarnessInstallFailed, SandboxError
|
||||
from litellm.harness.errors import HarnessError, HarnessInstallFailed, SandboxError
|
||||
from litellm.harness.handlers.base import BaseHarnessHandler
|
||||
from litellm.harness.sandbox.base import Process, Sandbox
|
||||
from litellm.harness.sandbox.base import CompletedRun, Process, Sandbox
|
||||
from litellm.harness.types import Event
|
||||
from litellm.llms.base_llm.harness.transformation import BaseCLIHarnessConfig, HarnessSessionSetup
|
||||
from litellm.llms.base_llm.harness.transformation import (
|
||||
BaseCLIHarnessConfig,
|
||||
HarnessSessionCopy,
|
||||
HarnessSessionSetup,
|
||||
HarnessTurnError,
|
||||
)
|
||||
from litellm.llms.base_llm.harness.utils import decode_json_line, read_skill_files
|
||||
|
||||
# Link <private_dir>/<dir> to a LiteLLM-owned cache dir so a later session can resume.
|
||||
|
|
@ -81,6 +91,52 @@ async def persist_dir(sandbox: Sandbox, link_path: str, cache_subpath: str) -> N
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ResumeFailed:
|
||||
reason: str
|
||||
|
||||
|
||||
async def bounded_resume_run(
|
||||
sandbox: Sandbox, cmd: Sequence[str], env: Mapping[str, str]
|
||||
) -> CompletedRun | ResumeFailed:
|
||||
try:
|
||||
run: Final = await sandbox.run(cmd, env=env, timeout=HARNESS_RESUME_COMMAND_TIMEOUT_SECONDS)
|
||||
except SandboxError as exc:
|
||||
return ResumeFailed(str(exc))
|
||||
return run if run.exit_code == 0 else ResumeFailed(run.stderr.strip() or f"exit code {run.exit_code}")
|
||||
|
||||
|
||||
async def import_session_copy(sandbox: Sandbox, copy: HarnessSessionCopy, env: Mapping[str, str]) -> str | ResumeFailed:
|
||||
try:
|
||||
await sandbox.write(copy.path, copy.data)
|
||||
except SandboxError as exc:
|
||||
return ResumeFailed(str(exc))
|
||||
imported: Final = await bounded_resume_run(sandbox, copy.import_command, env)
|
||||
return imported if isinstance(imported, ResumeFailed) else copy.session_id
|
||||
|
||||
|
||||
async def resumable_session_id(
|
||||
config: BaseCLIHarnessConfig[object, object],
|
||||
sandbox: Sandbox,
|
||||
setup_env: Mapping[str, str],
|
||||
private_dir: str,
|
||||
native_session_id: str,
|
||||
) -> str | ResumeFailed:
|
||||
export_cmd: Final = config.transform_resume_export_command(native_session_id)
|
||||
if export_cmd is None:
|
||||
return native_session_id
|
||||
exported: Final = await bounded_resume_run(sandbox, export_cmd, setup_env)
|
||||
if isinstance(exported, ResumeFailed):
|
||||
return exported
|
||||
try:
|
||||
copy: Final = config.transform_resume_copy(exported.stdout, private_dir, sandbox.workdir)
|
||||
except HarnessError as exc:
|
||||
return ResumeFailed(str(exc))
|
||||
if copy is None:
|
||||
return native_session_id
|
||||
return await import_session_copy(sandbox, copy, setup_env)
|
||||
|
||||
|
||||
async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str) -> None:
|
||||
for skill in skills:
|
||||
name = os.path.basename(os.path.realpath(os.fspath(skill)))
|
||||
|
|
@ -89,13 +145,14 @@ async def copy_skills(sandbox: Sandbox, skills: Sequence[str], skills_root: str)
|
|||
|
||||
|
||||
class CLIHarnessHandler(BaseHarnessHandler):
|
||||
config: BaseCLIHarnessConfig
|
||||
config: BaseCLIHarnessConfig[object, object]
|
||||
|
||||
def __init__(self, config: BaseCLIHarnessConfig) -> None:
|
||||
super().__init__(config)
|
||||
self._private_dir: str | None = None
|
||||
self._setup: HarnessSessionSetup | None = None
|
||||
self._native_id: str | None = None
|
||||
self._pending_resume: str | None = None
|
||||
self._proc: Process | None = None
|
||||
|
||||
async def start(self, ctx: SessionContext) -> None:
|
||||
|
|
@ -119,6 +176,9 @@ class CLIHarnessHandler(BaseHarnessHandler):
|
|||
async def turn(self, ctx: SessionContext, prompt: str) -> AsyncIterator[Event]:
|
||||
if self._setup is None or self._private_dir is None:
|
||||
raise RuntimeError("CLIHarnessHandler.turn() called before start()")
|
||||
if self._pending_resume is not None:
|
||||
self._native_id = await self._prepared_session_id(ctx, self._setup, self._private_dir, self._pending_resume)
|
||||
self._pending_resume = None
|
||||
request = self.config.transform_turn_request(ctx, self._setup, self._private_dir, prompt, self._native_id)
|
||||
argv: Final = list(request.argv) # mutable-ok: Sandbox.exec takes list[str]
|
||||
proc = await ctx.sandbox.exec(argv, env=request.env, cwd=request.cwd)
|
||||
|
|
@ -158,4 +218,16 @@ class CLIHarnessHandler(BaseHarnessHandler):
|
|||
return self._native_id
|
||||
|
||||
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
|
||||
self._native_id = native_session_id
|
||||
if native_session_id != self._native_id:
|
||||
self._native_id = native_session_id
|
||||
self._pending_resume = native_session_id
|
||||
|
||||
async def _prepared_session_id(
|
||||
self, ctx: SessionContext, setup: HarnessSessionSetup, private_dir: str, saved_id: str
|
||||
) -> str:
|
||||
target: Final = await resumable_session_id(self.config, ctx.sandbox, setup.env, private_dir, saved_id)
|
||||
match target:
|
||||
case ResumeFailed(reason=reason):
|
||||
raise HarnessTurnError(f"could not prepare session {saved_id} for resume: {reason}")
|
||||
case session_id:
|
||||
return session_id
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
|
|||
break
|
||||
resume: dict[str, object] = {} # mutable-ok: Command(resume=) payload, filled per answered approval
|
||||
for interrupt in state.interrupts:
|
||||
decisions: list[dict[str, Any]] = [] # mutable-ok: HITL decisions collected across awaited approvals
|
||||
decisions: list[dict[str, str]] = [] # mutable-ok: HITL decisions collected across awaited approvals
|
||||
for request in approval_requests(getattr(interrupt, "value", None)):
|
||||
approval = Approval(
|
||||
tool=normalized_tool_name(str(request.get("name") or "")),
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from typing import Final
|
|||
|
||||
from litellm.constants import HARNESS_PROCESS_KILL_GRACE_SECONDS
|
||||
from litellm.harness.errors import SandboxError
|
||||
from litellm.harness.sandbox.base import CompletedRun
|
||||
from litellm.harness.sandbox.base import CompletedRun, Process
|
||||
from litellm.harness.sandbox.snapshot import snapshot_local
|
||||
|
||||
_SECRET_PREFIXES: Final = (
|
||||
|
|
@ -103,7 +103,7 @@ class SubprocessHandle:
|
|||
await self._proc.wait()
|
||||
|
||||
|
||||
async def _read_all(handle: SubprocessHandle) -> tuple[bytes, bytes, int]:
|
||||
async def _read_all(handle: Process) -> tuple[bytes, bytes, int]:
|
||||
if handle.stdin is not None:
|
||||
handle.stdin.close()
|
||||
stdout, stderr = await asyncio.gather(handle.stdout.read(), handle.stderr.read())
|
||||
|
|
@ -111,13 +111,16 @@ async def _read_all(handle: SubprocessHandle) -> tuple[bytes, bytes, int]:
|
|||
return stdout, stderr, exit_code
|
||||
|
||||
|
||||
async def collect_output(handle: SubprocessHandle, cmd: Sequence[str], timeout: float | None) -> CompletedRun:
|
||||
"""Close stdin, read stdout/stderr to EOF; kill and raise SandboxError on timeout."""
|
||||
async def collect_output(handle: Process, cmd: Sequence[str], timeout: float | None) -> CompletedRun:
|
||||
"""Close stdin, read stdout/stderr to EOF; kill on cancel, and kill and raise SandboxError on timeout."""
|
||||
try:
|
||||
stdout, stderr, code = await asyncio.wait_for(_read_all(handle), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await handle.kill()
|
||||
raise SandboxError(f"command timed out after {timeout}s: {cmd[0]}")
|
||||
except asyncio.CancelledError:
|
||||
await handle.kill()
|
||||
raise
|
||||
return CompletedRun(
|
||||
stdout=stdout.decode("utf-8", errors="replace"),
|
||||
stderr=stderr.decode("utf-8", errors="replace"),
|
||||
|
|
@ -180,14 +183,15 @@ class LocalSandbox:
|
|||
self._check_open()
|
||||
if not cmd:
|
||||
raise SandboxError("exec() needs a non-empty command")
|
||||
work_dir: Final = self._resolve_cwd(cwd)
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdin=asyncio.subprocess.PIPE,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=self._resolve_cwd(cwd),
|
||||
env=self.child_env(env),
|
||||
cwd=work_dir,
|
||||
env=self.child_env(MappingProxyType({**(env or {}), "PWD": work_dir})),
|
||||
start_new_session=True,
|
||||
)
|
||||
except (FileNotFoundError, PermissionError) as exc:
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ way (`pod_lock`, `budget_reset`, `spend_queue`, `health_check`, `scheduler_queue
|
|||
rather than a bare `redis.set`; `tests/unit/test_internal_context.py` scans every
|
||||
module under `litellm/` and `enterprise/` that calls a shared cache or declares a
|
||||
read or write on the request batch (`reserve_redis_batch_reads`,
|
||||
`declare_batch_get`, `batch.mget`, `batch.set`, `batch.script`) and fails when
|
||||
`batch.mget`, `batch.set`, `batch.script`) and fails when
|
||||
one has no declared family, with the process-local `InMemoryCache` callers listed
|
||||
as the only exemptions. A batch op carries the family that was active when it was
|
||||
declared, so the routing prefetch armed before deployment selection
|
||||
|
|
|
|||
|
|
@ -214,6 +214,7 @@ _SERVICE_VERB_BY_CALL_TYPE: Final[dict[str, str]] = {
|
|||
"increment_cache": "incr",
|
||||
"async_increment": "incr",
|
||||
"async_increment_pipeline": "incr",
|
||||
"async_increment_if_present": "incr",
|
||||
"delete_cache": "delete",
|
||||
"async_delete_cache": "delete",
|
||||
"async_rpush": "rpush",
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from datetime import datetime
|
|||
from typing import Any, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import STREAM_SSE_DONE_STRING
|
||||
|
|
@ -25,6 +26,10 @@ from litellm.types.interactions import (
|
|||
)
|
||||
from litellm.utils import CustomStreamWrapper
|
||||
|
||||
_INTERACTIONS_STREAMING_CHUNK_ADAPTER: Final = TypeAdapter(
|
||||
object, config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
||||
|
||||
class BaseInteractionsAPIStreamingIterator:
|
||||
"""
|
||||
|
|
@ -83,7 +88,7 @@ class BaseInteractionsAPIStreamingIterator:
|
|||
|
||||
try:
|
||||
# Parse the JSON chunk
|
||||
parsed_chunk: Final = json.loads(stripped_chunk)
|
||||
parsed_chunk: Final = _INTERACTIONS_STREAMING_CHUNK_ADAPTER.validate_python(json.loads(stripped_chunk))
|
||||
|
||||
# Format as InteractionsAPIStreamingResponse
|
||||
if isinstance(parsed_chunk, dict):
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
|
|||
"aws_external_id",
|
||||
"aws_session_tags",
|
||||
"aws_bedrock_runtime_endpoint",
|
||||
"aws_bedrock_control_plane_endpoint",
|
||||
"aws_bedrock_project_id",
|
||||
}
|
||||
)
|
||||
|
|
@ -76,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"otpm",
|
||||
"use_xai_oauth",
|
||||
"github_copilot_auth_type",
|
||||
"github_copilot_host",
|
||||
"github_copilot_user_session",
|
||||
"fireworks_forward_user_id",
|
||||
"responses_api_version",
|
||||
|
|
|
|||
|
|
@ -117,7 +117,8 @@ from litellm.litellm_core_utils.ptu_pricing import is_spilled_over_ptu_request
|
|||
from litellm.litellm_core_utils.redact_messages import (
|
||||
redact_message_input_output_from_custom_logger,
|
||||
redact_message_input_output_from_logging,
|
||||
redact_streaming_responses_for_custom_logger,
|
||||
redacted_model_call_details_for_custom_logger,
|
||||
redacted_response_obj_for_custom_logger,
|
||||
should_redact_message_logging,
|
||||
)
|
||||
from litellm.litellm_core_utils.served_output_texts import (
|
||||
|
|
@ -337,6 +338,7 @@ def _get_provider_request_id(original_exception: Exception) -> str | None:
|
|||
|
||||
# Cache custom pricing keys as frozenset for O(1) lookups instead of looping through 49 keys
|
||||
_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields.keys())
|
||||
_METADATA_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_CUSTOM_PRICING_KEYS: Final[frozenset[str]] = _CUSTOM_PRICING_KEYS | DEPLOYMENT_SCOPED_PRICING_FIELDS
|
||||
_UNSERIALIZABLE_METADATA_KEYS: Final[frozenset[str]] = frozenset(
|
||||
("user_api_key_auth", "user_api_key_budget_reservation")
|
||||
|
|
@ -794,6 +796,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.defer_async_logging: bool = False
|
||||
self.enqueue_deferred_logging: Callable[[], None] | None = None
|
||||
self._async_success_scheduled: bool = False
|
||||
self._async_success_enqueued: bool = False
|
||||
self._sync_success_scheduled: bool = False
|
||||
self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
|
||||
self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
|
||||
|
|
@ -995,6 +998,21 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return model_id
|
||||
return None
|
||||
|
||||
def get_router_pricing_provider(self) -> str | None:
|
||||
"""The deployment's ``model_info.pricing_provider``, the provider its cost-map lookups run under."""
|
||||
litellm_params: Final = cast( # cast-ok: callback metadata is caller-provided
|
||||
Mapping[str, object] | None, getattr(self, "litellm_params", None)
|
||||
)
|
||||
if not litellm_params:
|
||||
return None
|
||||
candidates: Final = (
|
||||
_METADATA_MAPPING.validate_python(
|
||||
_METADATA_MAPPING.validate_python(litellm_params.get(key) or {}).get("model_info") or {}
|
||||
).get("pricing_provider")
|
||||
for key in ("litellm_metadata", "metadata")
|
||||
)
|
||||
return next((provider for provider in candidates if isinstance(provider, str) and provider), None)
|
||||
|
||||
def _routed_deployment_model(self) -> str | None:
|
||||
litellm_params: Final = self._model_call_details_view.get("litellm_params")
|
||||
metadata: Final = (
|
||||
|
|
@ -2020,6 +2038,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
"prompt": prompt,
|
||||
"standard_built_in_tools_params": self.standard_built_in_tools_params,
|
||||
"router_model_id": router_model_id,
|
||||
"pricing_provider": self.get_router_pricing_provider(),
|
||||
"litellm_logging_obj": self,
|
||||
"data_residency": (
|
||||
self.litellm_params.get("data_residency")
|
||||
|
|
@ -2164,7 +2183,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
pricing: Final = pricing_entry_for_cost_calc(
|
||||
model=model,
|
||||
completion_response=result,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
custom_llm_provider=self.get_router_pricing_provider() or custom_llm_provider,
|
||||
custom_pricing=self.custom_pricing_for(result),
|
||||
base_model=get_base_model_from_metadata(model_call_details=self.model_call_details),
|
||||
router_model_id=router_model_id or self.get_router_model_id(),
|
||||
|
|
@ -2190,20 +2209,23 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
usage=usage,
|
||||
)
|
||||
|
||||
def custom_pricing_for(self, result: object) -> bool:
|
||||
litellm_params: Final = getattr(self, "litellm_params", None)
|
||||
def spilled_over_ptu_for(self, result: object) -> bool:
|
||||
result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({})
|
||||
additional_headers: Final = (
|
||||
result_hidden_params.get("additional_headers")
|
||||
if isinstance(result_hidden_params, dict)
|
||||
else getattr(result_hidden_params, "additional_headers", None)
|
||||
)
|
||||
spilled_over: Final = is_spilled_over_ptu_request(
|
||||
model_info=_deployment_model_info(litellm_params),
|
||||
return is_spilled_over_ptu_request(
|
||||
model_info=_deployment_model_info(getattr(self, "litellm_params", None)),
|
||||
response_headers=self.model_call_details.get("response_headers"),
|
||||
additional_headers=additional_headers,
|
||||
)
|
||||
return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params)
|
||||
|
||||
def custom_pricing_for(self, result: object) -> bool:
|
||||
if self.spilled_over_ptu_for(result):
|
||||
return False
|
||||
return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None))
|
||||
|
||||
def _prompt_for_cost_calculation(self) -> str:
|
||||
"""
|
||||
|
|
@ -2447,7 +2469,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
await self.async_failure_handler(exception, traceback_exception)
|
||||
finally:
|
||||
if self._should_run_sync_failure_callbacks_for_async_calls():
|
||||
executor.submit(self.failure_handler, exception, traceback_exception)
|
||||
executor.submit(self.failure_handler, exception, traceback_exception, for_async_request=True)
|
||||
|
||||
def should_run_logging(
|
||||
self,
|
||||
|
|
@ -2462,6 +2484,16 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
except Exception:
|
||||
return True
|
||||
|
||||
@property
|
||||
def async_success_log_enqueued(self) -> bool:
|
||||
"""True once the success log task was actually created, not merely claimed, so a later failure raised
|
||||
around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second
|
||||
time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False."""
|
||||
return self._async_success_enqueued
|
||||
|
||||
def mark_async_success_log_enqueued(self) -> None:
|
||||
self._async_success_enqueued = True
|
||||
|
||||
def claim_async_success_log(self) -> bool:
|
||||
"""One async success log per request. The innermost @client wrapper always exits first,
|
||||
since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
|
||||
|
|
@ -3489,8 +3521,10 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
): # custom logger class
|
||||
if self.stream and complete_streaming_response is None:
|
||||
callback.log_stream_event(
|
||||
kwargs=call_details,
|
||||
response_obj=result,
|
||||
kwargs=redacted_model_call_details_for_custom_logger(
|
||||
model_call_details=call_details, custom_logger=callback
|
||||
),
|
||||
response_obj=redacted_response_obj_for_custom_logger(result, callback),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
|
|
@ -3500,8 +3534,13 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
result = call_details_view["complete_response"]
|
||||
|
||||
callback.log_success_event(
|
||||
kwargs=callback.redact_standard_logging_payload_from_model_call_details(call_details),
|
||||
response_obj=result,
|
||||
kwargs=redacted_model_call_details_for_custom_logger(
|
||||
model_call_details=callback.redact_standard_logging_payload_from_model_call_details(
|
||||
call_details
|
||||
),
|
||||
custom_logger=callback,
|
||||
),
|
||||
response_obj=redacted_response_obj_for_custom_logger(result, callback),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
|
|
@ -3844,7 +3883,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
model_call_details = callback.redact_standard_logging_payload_from_model_call_details(
|
||||
model_call_details=model_call_details
|
||||
)
|
||||
model_call_details = redact_streaming_responses_for_custom_logger(
|
||||
model_call_details = redacted_model_call_details_for_custom_logger(
|
||||
model_call_details=model_call_details, custom_logger=callback
|
||||
)
|
||||
##################################
|
||||
|
|
@ -3855,7 +3894,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
kwargs=model_call_details,
|
||||
response_obj=model_call_details["async_complete_streaming_response"]
|
||||
if self.stream
|
||||
else result,
|
||||
else redacted_response_obj_for_custom_logger(result, callback),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
guardrail_snapshot=self.shadow_eval_request_snapshot,
|
||||
|
|
@ -3871,14 +3910,14 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
else:
|
||||
await callback.async_log_stream_event( # [TODO]: move this to being an async log stream event function
|
||||
kwargs=model_call_details,
|
||||
response_obj=result,
|
||||
response_obj=redacted_response_obj_for_custom_logger(result, callback),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
else:
|
||||
await callback.async_log_success_event(
|
||||
kwargs=model_call_details,
|
||||
response_obj=result,
|
||||
response_obj=redacted_response_obj_for_custom_logger(result, callback),
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
)
|
||||
|
|
@ -4054,15 +4093,21 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
traceback_exception: str,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
*,
|
||||
for_async_request: bool = False,
|
||||
) -> None:
|
||||
"""Restores trace_id/session_id contextvars once this attempt's own failure
|
||||
logging (including any nested calls its callbacks trigger) is fully done."""
|
||||
logging (including any nested calls its callbacks trigger) is fully done.
|
||||
|
||||
``for_async_request`` marks the sync run of a failure whose async run also logs it, so a
|
||||
CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook."""
|
||||
try:
|
||||
return self._failure_handler_body(
|
||||
exception=exception,
|
||||
traceback_exception=traceback_exception,
|
||||
start_time=start_time,
|
||||
end_time=end_time,
|
||||
for_async_request=for_async_request,
|
||||
)
|
||||
finally:
|
||||
self._restore_correlation_context()
|
||||
|
|
@ -4073,6 +4118,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
traceback_exception: str,
|
||||
start_time: datetime.datetime | None = None,
|
||||
end_time: datetime.datetime | None = None,
|
||||
for_async_request: bool = False,
|
||||
) -> None:
|
||||
verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback)
|
||||
if not self.should_run_logging(event_type="sync_failure"): # prevent double logging
|
||||
|
|
@ -4165,6 +4211,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
isinstance(callback, CustomLogger)
|
||||
and is_sync_request
|
||||
and self.call_type != CallTypes.pass_through.value
|
||||
and not (for_async_request and self._async_handler_logs_to(callback))
|
||||
): # custom logger class
|
||||
callback.log_failure_event(
|
||||
start_time=start_time,
|
||||
|
|
@ -4527,6 +4574,12 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
cb_name: Final = self._get_callback_name(cb)
|
||||
return any(prefix in cb_name for prefix in INTERNAL_PREFIXES)
|
||||
|
||||
def _async_handler_logs_to(self, logger: CustomLogger) -> bool:
|
||||
"""True when the async failure run of this call hands ``logger`` its own ``async_log_failure_event``."""
|
||||
if type(logger).async_log_failure_event is CustomLogger.async_log_failure_event: # pyright: ignore[reportUnknownMemberType] # CustomLogger's hook signatures are untyped
|
||||
return False
|
||||
return logger in (self.dynamic_async_failure_callbacks or []) or logger in litellm._async_failure_callback # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # callback lists are untyped and have no public accessor
|
||||
|
||||
def _remove_internal_custom_logger_callbacks(self, callbacks: list) -> list:
|
||||
"""
|
||||
Removes internal custom logger callbacks from the list.
|
||||
|
|
@ -7195,13 +7248,16 @@ def get_standard_logging_object_payload(
|
|||
saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0
|
||||
|
||||
## Get model cost information ##
|
||||
base_model = get_base_model_from_metadata(model_call_details=kwargs)
|
||||
# The router overrides completion_response.model to the model-group alias before
|
||||
# this payload is built, so cost-map lookup via that alias always misses.
|
||||
# Fall back to the actual deployment model set by the router in metadata.
|
||||
if base_model is None:
|
||||
base_model = metadata.get("deployment")
|
||||
custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj)
|
||||
configured_base_model: Final = get_base_model_from_metadata(model_call_details=kwargs)
|
||||
spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj)
|
||||
# The router overrides completion_response.model to the model-group alias before this payload is
|
||||
# built, so cost-map lookup via that alias misses and the deployment model stands in for base_model.
|
||||
# A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates,
|
||||
# which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there.
|
||||
base_model: Final = (
|
||||
metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model
|
||||
)
|
||||
custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj)
|
||||
raw_response_cost: Final = kwargs.get("response_cost")
|
||||
llm_response_cost: Final[float] = raw_response_cost or 0.0
|
||||
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ from functools import reduce
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.constants import LITELLM_DETAILED_TIMING
|
||||
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers
|
||||
|
|
@ -101,6 +102,32 @@ def response_timing_metrics(
|
|||
return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms}
|
||||
|
||||
|
||||
_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]:
|
||||
try:
|
||||
return _HIDDEN_PARAMS_DICT.validate_python(hidden_params)
|
||||
except ValidationError:
|
||||
return {}
|
||||
|
||||
|
||||
def _existing_provider_response_model(result: object) -> str | None:
|
||||
"""The pricing model a response already carries, like the decisions path sets, wins over the response model."""
|
||||
hidden_params: Final[object] = getattr(result, "_hidden_params", None)
|
||||
carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr(
|
||||
hidden_params, "provider_response_model", None
|
||||
)
|
||||
return carried if isinstance(carried, str) and carried else None
|
||||
|
||||
|
||||
def _provider_response_model(result: object) -> str | None:
|
||||
"""The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the
|
||||
client asked for and cost-map lookups for the standard logging payload would otherwise miss."""
|
||||
model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None
|
||||
return model if isinstance(model, str) and model else None
|
||||
|
||||
|
||||
class ResponseMetadata:
|
||||
"""
|
||||
Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses
|
||||
|
|
@ -147,6 +174,8 @@ class ResponseMetadata:
|
|||
preserve_litellm_internal_headers=True,
|
||||
),
|
||||
"litellm_model_name": model,
|
||||
"provider_response_model": _existing_provider_response_model(self.result)
|
||||
or _provider_response_model(self.result),
|
||||
}
|
||||
self._update_hidden_params(new_params)
|
||||
|
||||
|
|
|
|||
|
|
@ -10,10 +10,12 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import inspect
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -40,36 +42,79 @@ else:
|
|||
LiteLLMLoggingObject = Any
|
||||
|
||||
|
||||
def _opted_out_of_message_logging(custom_logger: CustomLogger) -> bool:
|
||||
return hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True
|
||||
|
||||
|
||||
def redact_message_input_output_from_custom_logger(
|
||||
litellm_logging_obj: LiteLLMLoggingObject, result, custom_logger: CustomLogger
|
||||
):
|
||||
if hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True:
|
||||
) -> object:
|
||||
if _opted_out_of_message_logging(custom_logger):
|
||||
return perform_redaction(litellm_logging_obj.model_call_details, result, redact_streaming_responses=False)
|
||||
return result
|
||||
|
||||
|
||||
def redact_streaming_responses_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict:
|
||||
def redacted_model_call_details_for_custom_logger(model_call_details: dict, custom_logger: CustomLogger) -> dict:
|
||||
"""
|
||||
Returns a copy of model_call_details whose streaming response entries are redacted deepcopies
|
||||
when the custom logger has opted out of message logging. The shared model_call_details is left
|
||||
untouched so other callbacks still receive the unredacted response.
|
||||
Returns a copy of model_call_details with the request messages, instructions and streaming responses
|
||||
redacted when the custom logger has opted out of message logging. The shared model_call_details is left
|
||||
untouched so other callbacks and the spend log still receive the unredacted values.
|
||||
"""
|
||||
if not (hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True):
|
||||
if not _opted_out_of_message_logging(custom_logger):
|
||||
return model_call_details
|
||||
redacted_entries: Final = {
|
||||
streaming_key: _redacted_streaming_response_copy(model_call_details[streaming_key])
|
||||
for streaming_key in ("complete_streaming_response", "async_complete_streaming_response")
|
||||
if model_call_details.get(streaming_key) is not None
|
||||
return {
|
||||
**{key: value for key, value in model_call_details.items() if key != SERVED_OUTPUT_TEXTS_KEY},
|
||||
**_redacted_entries_for_opted_out_logger(model_call_details),
|
||||
}
|
||||
if not redacted_entries:
|
||||
return model_call_details
|
||||
return {**model_call_details, **redacted_entries}
|
||||
|
||||
|
||||
def _redacted_streaming_response_copy(streaming_response):
|
||||
redacted_response: Final = copy.deepcopy(streaming_response)
|
||||
_redact_streaming_response(redacted_response)
|
||||
return redacted_response
|
||||
def _redacted_entries_for_opted_out_logger(model_call_details: Mapping[str, object]) -> Mapping[str, object]:
|
||||
optional_params: Final = _as_mapping(model_call_details.get("optional_params"))
|
||||
litellm_params: Final = _as_mapping(model_call_details.get("litellm_params"))
|
||||
standard_logging_object: Final = _as_mapping(model_call_details.get("standard_logging_object"))
|
||||
return {
|
||||
"messages": [{"role": "user", "content": REDACTED_BY_LITELLM}],
|
||||
"prompt": "",
|
||||
"input": "",
|
||||
**({"instructions": REDACTED_BY_LITELLM} if model_call_details.get("instructions") is not None else {}),
|
||||
**({"optional_params": _with_redacted_instructions(optional_params)} if optional_params is not None else {}),
|
||||
**({"litellm_params": redacted_litellm_params(litellm_params)} if litellm_params is not None else {}),
|
||||
**(
|
||||
{"standard_logging_object": _redact_standard_logging_object(standard_logging_object)}
|
||||
if standard_logging_object is not None
|
||||
else {}
|
||||
),
|
||||
**{
|
||||
streaming_key: _redacted_streaming_response_copy(model_call_details[streaming_key])
|
||||
for streaming_key in (
|
||||
"complete_streaming_response",
|
||||
"async_complete_streaming_response",
|
||||
"complete_response",
|
||||
)
|
||||
if model_call_details.get(streaming_key) is not None
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def redacted_response_obj_for_custom_logger(response_obj: object, custom_logger: CustomLogger) -> object:
|
||||
if not _opted_out_of_message_logging(custom_logger) or not _is_redactable_response(response_obj):
|
||||
return response_obj
|
||||
return perform_redaction({}, response_obj)
|
||||
|
||||
|
||||
def _is_redactable_response(response_obj: object) -> bool:
|
||||
return isinstance(
|
||||
response_obj, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse)
|
||||
) or (isinstance(response_obj, dict) and ("choices" in response_obj or "output" in response_obj))
|
||||
|
||||
|
||||
def _redacted_streaming_response_copy(streaming_response: object) -> object:
|
||||
redacted_response: Final = (
|
||||
streaming_response
|
||||
if isinstance(streaming_response, litellm.ResponsesAPIResponse)
|
||||
else copy.deepcopy(streaming_response)
|
||||
)
|
||||
return _redact_streaming_response(redacted_response)
|
||||
|
||||
|
||||
def _redact_streaming_response(streaming_response):
|
||||
|
|
@ -77,10 +122,13 @@ def _redact_streaming_response(streaming_response):
|
|||
for choice in streaming_response.choices:
|
||||
_redact_choice_content(choice)
|
||||
redact_vertex_ai_metadata_from_logged_object(streaming_response)
|
||||
elif isinstance(streaming_response, litellm.ResponsesAPIResponse):
|
||||
return perform_redaction({}, streaming_response)
|
||||
elif hasattr(streaming_response, "output"):
|
||||
_redact_responses_api_output(streaming_response.output)
|
||||
if hasattr(streaming_response, "reasoning") and streaming_response.reasoning is not None:
|
||||
streaming_response.reasoning = None
|
||||
return streaming_response
|
||||
|
||||
|
||||
def _redact_tool_calls(tool_calls) -> None:
|
||||
|
|
@ -177,31 +225,75 @@ def _redact_responses_api_output(output_items):
|
|||
output_item.input = REDACTED_BY_LITELLM
|
||||
|
||||
|
||||
def _redact_responses_api_output_dict(output_items, redacted_str: str):
|
||||
"""Helper to redact ResponsesAPIResponse output items in dict form."""
|
||||
for output_item in output_items:
|
||||
if not isinstance(output_item, dict):
|
||||
continue
|
||||
_RESPONSE_ITEMS: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
if output_item.get("text") is not None:
|
||||
output_item["text"] = redacted_str
|
||||
|
||||
if isinstance(output_item.get("content"), list):
|
||||
for content_item in output_item["content"]:
|
||||
if isinstance(content_item, dict) and content_item.get("text") is not None:
|
||||
content_item["text"] = redacted_str
|
||||
if isinstance(content_item, dict) and content_item.get("refusal") is not None:
|
||||
content_item["refusal"] = redacted_str
|
||||
def _redacted_responses_api_output_part(part: object, fields: tuple[str, ...], redacted_str: str) -> object:
|
||||
if (part_mapping := _as_mapping(part)) is None:
|
||||
return part
|
||||
return {key: redacted_str if key in fields and value is not None else value for key, value in part_mapping.items()}
|
||||
|
||||
if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
|
||||
for summary_item in output_item["summary"]:
|
||||
if isinstance(summary_item, dict) and summary_item.get("text") is not None:
|
||||
summary_item["text"] = redacted_str
|
||||
|
||||
if output_item.get("type") == "function_call" and "arguments" in output_item:
|
||||
output_item["arguments"] = redacted_str
|
||||
if output_item.get("type") == "custom_tool_call" and "input" in output_item:
|
||||
output_item["input"] = redacted_str
|
||||
def _redacted_responses_api_output_item(item: object, redacted_str: str) -> object:
|
||||
if (item_mapping := _as_mapping(item)) is None:
|
||||
return item
|
||||
content: Final = item_mapping.get("content")
|
||||
summary: Final = item_mapping.get("summary")
|
||||
return {
|
||||
**item_mapping,
|
||||
**({"text": redacted_str} if item_mapping.get("text") is not None else {}),
|
||||
**(
|
||||
{
|
||||
"content": [
|
||||
_redacted_responses_api_output_part(part, ("text", "refusal"), redacted_str)
|
||||
for part in _RESPONSE_ITEMS.validate_python(content)
|
||||
]
|
||||
}
|
||||
if isinstance(content, list)
|
||||
else {}
|
||||
),
|
||||
**(
|
||||
{
|
||||
"summary": [
|
||||
_redacted_responses_api_output_part(part, ("text",), redacted_str)
|
||||
for part in _RESPONSE_ITEMS.validate_python(summary)
|
||||
]
|
||||
}
|
||||
if item_mapping.get("type") == "reasoning" and isinstance(summary, list)
|
||||
else {}
|
||||
),
|
||||
**(
|
||||
{"arguments": redacted_str}
|
||||
if item_mapping.get("type") == "function_call" and "arguments" in item_mapping
|
||||
else {}
|
||||
),
|
||||
**(
|
||||
{"input": redacted_str}
|
||||
if item_mapping.get("type") == "custom_tool_call" and "input" in item_mapping
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _redact_responses_api_output_dict(output_items: Sequence[object], redacted_str: str) -> Sequence[object]:
|
||||
return [_redacted_responses_api_output_item(item, redacted_str) for item in output_items]
|
||||
|
||||
|
||||
def _with_redacted_instructions(params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return params if params.get("instructions") is None else {**params, "instructions": REDACTED_BY_LITELLM}
|
||||
|
||||
|
||||
def _redacted_responses_api_response(
|
||||
response: Mapping[str, object], *, redact_instructions: bool = True
|
||||
) -> Mapping[str, object]:
|
||||
output: Final = response.get("output")
|
||||
redacted: Final = {
|
||||
**response,
|
||||
"output": _redact_responses_api_output_dict(_RESPONSE_ITEMS.validate_python(output), REDACTED_BY_LITELLM)
|
||||
if isinstance(output, list)
|
||||
else output,
|
||||
}
|
||||
return _with_redacted_instructions(redacted) if redact_instructions else redacted
|
||||
|
||||
|
||||
def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -215,20 +307,25 @@ def redacted_standard_logging_payload(payload: Mapping[str, object]) -> Mapping[
|
|||
return _redact_standard_logging_object(payload)
|
||||
|
||||
|
||||
def _redact_standard_logging_object(payload: Mapping[str, object]) -> dict[str, object]:
|
||||
def _redact_standard_logging_object(
|
||||
payload: Mapping[str, object], *, redact_instructions: bool = True
|
||||
) -> dict[str, object]:
|
||||
standard_logging_object: Final = copy.deepcopy(without_classifier_audit(payload))
|
||||
redacted_str: Final = REDACTED_BY_LITELLM
|
||||
|
||||
if standard_logging_object.get("messages") is not None:
|
||||
standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
|
||||
model_parameters: Final = _as_mapping(standard_logging_object.get("model_parameters"))
|
||||
if model_parameters is not None and redact_instructions:
|
||||
standard_logging_object["model_parameters"] = _with_redacted_instructions(model_parameters)
|
||||
|
||||
response: Final = standard_logging_object.get("response")
|
||||
if response is not None:
|
||||
if isinstance(response, dict) and "output" in response:
|
||||
# ResponsesAPIResponse format - redact content in output items
|
||||
if isinstance(response.get("output"), list):
|
||||
_redact_responses_api_output_dict(response["output"], redacted_str)
|
||||
redact_vertex_ai_metadata_from_logged_object(response)
|
||||
standard_logging_object["response"] = _redacted_responses_api_response(
|
||||
response, redact_instructions=redact_instructions
|
||||
)
|
||||
redact_vertex_ai_metadata_from_logged_object(standard_logging_object["response"])
|
||||
elif isinstance(response, dict) and "choices" in response:
|
||||
# ModelResponse dict format - redact content in choices
|
||||
if isinstance(response.get("choices"), list):
|
||||
|
|
@ -320,11 +417,11 @@ def _redact_hidden_provider_specific_fields(params: Mapping[str, object]) -> Non
|
|||
def redacted_litellm_params(params: Mapping[str, object], *, redact_request_body: bool = True) -> dict[str, object]:
|
||||
"""
|
||||
Copy of litellm_params without classifier audit fields or baseline route request parameters.
|
||||
redact_request_body also redacts the nested proxy_server_request body.
|
||||
redact_request_body also redacts the request instructions and the nested proxy_server_request body.
|
||||
"""
|
||||
request: Final = _as_mapping(params.get("proxy_server_request"))
|
||||
return {
|
||||
**params,
|
||||
**(_with_redacted_instructions(params) if redact_request_body else params),
|
||||
**{
|
||||
key: _redacted_baseline_metadata(value)
|
||||
for key, value in params.items()
|
||||
|
|
@ -344,8 +441,9 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
|
||||
redact_streaming_responses=False skips the in-place redaction of the shared streaming
|
||||
response entries; per-callback redaction hands each opted-out callback its own redacted
|
||||
copy via redact_streaming_responses_for_custom_logger instead. It also leaves the shared
|
||||
proxy request body and hidden provider fields as they are, since the spend log reads them.
|
||||
copy via redacted_model_call_details_for_custom_logger instead. It also leaves the shared
|
||||
request instructions, proxy request body and hidden provider fields as they are, since the
|
||||
spend log and the other callbacks read them.
|
||||
"""
|
||||
# Redact model_call_details
|
||||
params: Final = _as_mapping(model_call_details.get("litellm_params"))
|
||||
|
|
@ -358,16 +456,31 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
|
||||
model_call_details["prompt"] = ""
|
||||
model_call_details["input"] = ""
|
||||
optional_params: Final = _as_mapping(model_call_details.get("optional_params"))
|
||||
if redact_streaming_responses:
|
||||
model_call_details.update(
|
||||
{
|
||||
**({"instructions": REDACTED_BY_LITELLM} if model_call_details.get("instructions") is not None else {}),
|
||||
**({"optional_params": _with_redacted_instructions(optional_params)} if optional_params else {}),
|
||||
}
|
||||
)
|
||||
model_call_details.pop(SERVED_OUTPUT_TEXTS_KEY, None)
|
||||
standard_logging_object: Final = model_call_details.get("standard_logging_object")
|
||||
if isinstance(standard_logging_object, Mapping):
|
||||
model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object)
|
||||
model_call_details["standard_logging_object"] = _redact_standard_logging_object(
|
||||
standard_logging_object, redact_instructions=redact_streaming_responses
|
||||
)
|
||||
redact_vertex_ai_metadata_from_litellm_params(model_call_details)
|
||||
|
||||
# Redact streaming response
|
||||
if redact_streaming_responses and model_call_details.get("stream", False) is True:
|
||||
for _streaming_key in ("complete_streaming_response", "async_complete_streaming_response"):
|
||||
_redact_streaming_response(model_call_details.get(_streaming_key))
|
||||
model_call_details.update(
|
||||
{
|
||||
streaming_key: _redact_streaming_response(model_call_details[streaming_key])
|
||||
for streaming_key in ("complete_streaming_response", "async_complete_streaming_response")
|
||||
if model_call_details.get(streaming_key) is not None
|
||||
}
|
||||
)
|
||||
|
||||
# Redact result
|
||||
if result is not None:
|
||||
|
|
@ -381,10 +494,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
# For async objects, return a simple redacted response without deepcopy
|
||||
return {"text": REDACTED_BY_LITELLM}
|
||||
|
||||
if not (
|
||||
isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
|
||||
or (isinstance(result, dict) and ("choices" in result or "output" in result))
|
||||
):
|
||||
if not _is_redactable_response(result):
|
||||
return {"text": REDACTED_BY_LITELLM}
|
||||
|
||||
_result: Final = copy.deepcopy(result)
|
||||
|
|
@ -399,14 +509,18 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
_redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
|
||||
redact_vertex_ai_metadata_from_logged_object(_result)
|
||||
elif isinstance(_result, dict) and "output" in _result:
|
||||
if isinstance(_result.get("output"), list):
|
||||
_redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
|
||||
return _redacted_responses_api_response(_result, redact_instructions=redact_streaming_responses)
|
||||
elif isinstance(_result, litellm.ResponsesAPIResponse):
|
||||
if hasattr(_result, "output"):
|
||||
_redact_responses_api_output(_result.output)
|
||||
# Redact reasoning field in ResponsesAPIResponse
|
||||
if hasattr(_result, "reasoning") and _result.reasoning is not None:
|
||||
_result.reasoning = None
|
||||
return _result.model_copy(
|
||||
update={"instructions": REDACTED_BY_LITELLM}
|
||||
if redact_streaming_responses and _result.instructions is not None
|
||||
else {}
|
||||
)
|
||||
elif isinstance(_result, litellm.EmbeddingResponse):
|
||||
if hasattr(_result, "data") and _result.data is not None:
|
||||
_result.data = []
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ Pattern Overview:
|
|||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Mapping, MutableSequence, Sequence
|
||||
from collections.abc import Iterable, Mapping, MutableSequence, Sequence
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain, repeat
|
||||
|
|
@ -27,8 +27,10 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.pass_through.adapters.transformation import (
|
||||
OPENAI_MAX_TOOL_NAME_LENGTH,
|
||||
LiteLLMAnthropicMessagesAdapter,
|
||||
is_provider_native_tool_dict,
|
||||
truncate_tool_name,
|
||||
)
|
||||
from litellm.llms.anthropic.pass_through.convertible_blocks import (
|
||||
keep_non_string_tool_result_texts,
|
||||
|
|
@ -202,6 +204,60 @@ def _is_client_tool_use(block: Mapping[str, object]) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _tool_use_name(block: object) -> str | None:
|
||||
fields: Final = as_str_mapping(block)
|
||||
if fields is None or fields.get("type") != "tool_use":
|
||||
return None
|
||||
return anthropic_tool_name(fields)
|
||||
|
||||
|
||||
def _tool_use_names(message: object) -> tuple[str, ...]:
|
||||
return tuple(name for name in map(_tool_use_name, list_field(message, "content")) if name is not None)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OriginalToolNames:
|
||||
"""Caller tool names over the guardrail's limit, keyed by the shortened name the guardrail sees."""
|
||||
|
||||
declared: Mapping[str, str]
|
||||
used: Mapping[str, str]
|
||||
|
||||
|
||||
def _restorable_names(names: Iterable[str]) -> Mapping[str, str]:
|
||||
listed: Final = tuple(names)
|
||||
taken: Final = frozenset(listed)
|
||||
shortened: Final = tuple(
|
||||
(truncate_tool_name(name), name) for name in listed if len(name) > OPENAI_MAX_TOOL_NAME_LENGTH
|
||||
)
|
||||
return MappingProxyType({alias: name for alias, name in shortened if alias not in taken})
|
||||
|
||||
|
||||
def _original_tool_names(request: Mapping[str, object]) -> _OriginalToolNames:
|
||||
return _OriginalToolNames(
|
||||
declared=_restorable_names(chain.from_iterable(map(anthropic_tool_names, list_field(request, "tools")))),
|
||||
used=_restorable_names(chain.from_iterable(map(_tool_use_names, list_field(request, "messages")))),
|
||||
)
|
||||
|
||||
|
||||
def _restore_tool_use_names(messages: Sequence[object], original_names: Mapping[str, str]) -> None:
|
||||
for message in messages:
|
||||
for block in list_field(message, "content"):
|
||||
if (name := _tool_use_name(block)) is not None and name in original_names and isinstance(block, dict):
|
||||
block["name"] = original_names[name]
|
||||
|
||||
|
||||
def _with_original_name(tool: ChatCompletionToolParam, original_names: Mapping[str, str]) -> ChatCompletionToolParam:
|
||||
if "function" not in tool:
|
||||
return tool
|
||||
function: Final = tool["function"]
|
||||
fields: Final = as_str_mapping(function)
|
||||
name: Final = None if fields is None else fields.get("name")
|
||||
original: Final = original_names.get(name) if isinstance(name, str) else None
|
||||
if original is None:
|
||||
return tool
|
||||
return {**tool, "function": {**function, "name": original}}
|
||||
|
||||
|
||||
def _write_back_system_block(system: object, block_idx: int, response: str) -> None:
|
||||
if not isinstance(system, list):
|
||||
return
|
||||
|
|
@ -629,6 +685,9 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
model: Final = data.get("model")
|
||||
if model:
|
||||
inputs["model"] = model
|
||||
original_tool_names: Final = _original_tool_names(
|
||||
cast(Mapping[str, object], data) # cast-ok: Anthropic request body
|
||||
)
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=data,
|
||||
|
|
@ -643,7 +702,9 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
anthropic_config: Final = AnthropicConfig()
|
||||
anthropic_tools: Final[list[AllAnthropicToolsValues]] = []
|
||||
for tool in guardrailed_tools:
|
||||
converted_tool, mcp_server = anthropic_config.map_tool_helper(tool)
|
||||
converted_tool, mcp_server = anthropic_config.map_tool_helper(
|
||||
_with_original_name(tool, original_tool_names.declared)
|
||||
)
|
||||
if converted_tool is not None:
|
||||
anthropic_tools.append(converted_tool)
|
||||
# Note: MCP servers are handled separately in the main transformation
|
||||
|
|
@ -676,6 +737,7 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
),
|
||||
hoisted_system_message=hoisted_system_message,
|
||||
preserve_system_messages=has_midturn_system_message,
|
||||
original_tool_names=original_tool_names.used,
|
||||
)
|
||||
else:
|
||||
if guardrailed_texts and len(guardrailed_texts) != len(scanned):
|
||||
|
|
@ -840,6 +902,8 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
structured_messages: list, # mutable-ok: API message payload
|
||||
hoisted_system_message: object = None,
|
||||
preserve_system_messages: bool = False,
|
||||
*,
|
||||
original_tool_names: Mapping[str, str],
|
||||
) -> None:
|
||||
"""Write a guardrail's structured-message rewrite back without losing corrections."""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
|
|
@ -903,6 +967,10 @@ class AnthropicMessagesHandler(BaseTranslation):
|
|||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "thinking":
|
||||
block.pop("cache_control", None)
|
||||
_restore_tool_use_names(
|
||||
cast(Sequence[object], converted), # cast-ok: Anthropic message payload
|
||||
original_tool_names,
|
||||
)
|
||||
data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages"))
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -712,7 +712,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
|
|||
optional_params: dict,
|
||||
model: str,
|
||||
param: str,
|
||||
value: Any,
|
||||
value: object,
|
||||
drop_params: bool,
|
||||
output_key: str,
|
||||
) -> None:
|
||||
|
|
@ -1788,6 +1788,21 @@ def strip_thinking_blocks_from_anthropic_messages_request_dict(
|
|||
data.pop("thinking", None)
|
||||
|
||||
|
||||
_DROP_ANTHROPIC_MESSAGE: Final = object()
|
||||
|
||||
|
||||
def _strip_empty_content_blocks_from_message(message: object) -> object:
|
||||
if not isinstance(message, dict) or not isinstance(message.get("content"), list):
|
||||
return message
|
||||
content: Final = message["content"]
|
||||
filtered: Final = [
|
||||
block for block in content if not _is_empty_text_block(block) and not is_empty_unsigned_thinking_block(block)
|
||||
]
|
||||
if len(filtered) == len(content):
|
||||
return message
|
||||
return {**message, "content": filtered} if filtered else _DROP_ANTHROPIC_MESSAGE
|
||||
|
||||
|
||||
def strip_empty_content_blocks_from_anthropic_messages(
|
||||
messages: Sequence[object],
|
||||
) -> list[Any]:
|
||||
|
|
@ -1822,18 +1837,11 @@ def strip_empty_content_blocks_from_anthropic_messages(
|
|||
The caller's list and its content blocks are never mutated; modified
|
||||
messages are returned as shallow copies with a fresh content list.
|
||||
"""
|
||||
out: Final[list[Any]] = []
|
||||
for m in messages:
|
||||
if not isinstance(m, dict) or not isinstance(m.get("content"), list):
|
||||
out.append(m)
|
||||
continue
|
||||
content = m["content"]
|
||||
filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_unsigned_thinking_block(b)]
|
||||
if len(filtered) == len(content):
|
||||
out.append(m)
|
||||
elif filtered:
|
||||
out.append({**m, "content": filtered})
|
||||
return out
|
||||
return [
|
||||
stripped_message
|
||||
for message in messages
|
||||
if (stripped_message := _strip_empty_content_blocks_from_message(message)) is not _DROP_ANTHROPIC_MESSAGE
|
||||
]
|
||||
|
||||
|
||||
def _is_empty_text_block(block: object) -> bool:
|
||||
|
|
@ -1921,18 +1929,15 @@ def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> l
|
|||
and ``:`` — valid on the upstream provider but rejected by Anthropic when
|
||||
the session is switched to a native Anthropic deployment.
|
||||
"""
|
||||
out: Final[list[Any]] = []
|
||||
for m in messages:
|
||||
if not isinstance(m, dict) or not isinstance(m.get("content"), list):
|
||||
out.append(m)
|
||||
continue
|
||||
content = m["content"]
|
||||
new_content = [_sanitize_tool_use_id_content_block(b) for b in content]
|
||||
if new_content == content:
|
||||
out.append(m)
|
||||
else:
|
||||
out.append({**m, "content": new_content})
|
||||
return out
|
||||
return [_sanitize_tool_use_ids_in_message(message) for message in messages]
|
||||
|
||||
|
||||
def _sanitize_tool_use_ids_in_message(message: object) -> object:
|
||||
if not isinstance(message, dict) or not isinstance(message.get("content"), list):
|
||||
return message
|
||||
content: Final = message["content"]
|
||||
new_content: Final = [_sanitize_tool_use_id_content_block(block) for block in content]
|
||||
return message if new_content == content else {**message, "content": new_content}
|
||||
|
||||
|
||||
class _ReplayedSearchQuery(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ How it works:
|
|||
"""
|
||||
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -48,7 +48,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
|
|||
|
||||
def can_handle(
|
||||
self,
|
||||
tools: list[dict] | None,
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
custom_llm_provider: str | None,
|
||||
) -> bool:
|
||||
if not tools:
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Mapping, Sequence
|
||||
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
|
|
@ -21,7 +21,7 @@ class MessagesInterceptor(ABC):
|
|||
@abstractmethod
|
||||
def can_handle(
|
||||
self,
|
||||
tools: list[dict] | None,
|
||||
tools: Sequence[Mapping[str, object]] | None,
|
||||
custom_llm_provider: str | None,
|
||||
) -> bool:
|
||||
"""Return True if this interceptor should handle the request."""
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
## Translates OpenAI call to Anthropic `/v1/messages` format
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from collections.abc import AsyncGenerator, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import ConfigDict, field_validator
|
||||
|
|
@ -21,7 +21,10 @@ from litellm.llms.anthropic.pass_through.messages.utils import (
|
|||
)
|
||||
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
|
||||
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
|
||||
from litellm.responses.streaming_iterator import stream_error_status_and_message
|
||||
from litellm.responses.streaming_iterator import (
|
||||
partial_responses_incomplete_event_on_disconnect,
|
||||
stream_error_status_and_message,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -33,6 +36,7 @@ from .transformation import (
|
|||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.anthropic.pass_through.safeguards import SafeguardsEvaluator
|
||||
from litellm.types.llms.openai import ResponseIncompleteEvent
|
||||
|
||||
|
||||
class _UpstreamFailure(LiteLLMBaseModel):
|
||||
|
|
@ -488,8 +492,38 @@ class AnthropicResponsesStreamWrapper:
|
|||
|
||||
raise StopAsyncIteration
|
||||
|
||||
async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]:
|
||||
"""Yield SSE-encoded bytes for each Anthropic event chunk."""
|
||||
def async_anthropic_sse_wrapper(self) -> "AnthropicResponsesSSEByteStream":
|
||||
"""SSE-encoded bytes for each Anthropic event chunk."""
|
||||
return AnthropicResponsesSSEByteStream(self)
|
||||
|
||||
async def _sse_frames(self) -> AsyncGenerator[bytes, None]:
|
||||
safeguard_results: Final = StreamedSafeguardResults(self.safeguards_evaluator)
|
||||
async for chunk in self:
|
||||
yield anthropic_sse_frame(await safeguard_results.observe(chunk))
|
||||
|
||||
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
|
||||
return partial_responses_incomplete_event_on_disconnect(self.responses_stream)
|
||||
|
||||
|
||||
class AnthropicResponsesSSEByteStream:
|
||||
"""
|
||||
The byte stream the proxy relays for a default-route /v1/messages request.
|
||||
Keeps the Responses API iterator reachable so a client disconnect can bill
|
||||
the partial output the way native /v1/responses streams do.
|
||||
"""
|
||||
|
||||
def __init__(self, wrapper: AnthropicResponsesStreamWrapper) -> None:
|
||||
self._wrapper: Final = wrapper
|
||||
self._frames: Final = wrapper._sse_frames() # pyright: ignore[reportPrivateUsage] # the wrapper's own byte stream
|
||||
|
||||
def __aiter__(self) -> "AnthropicResponsesSSEByteStream":
|
||||
return self
|
||||
|
||||
async def __anext__(self) -> bytes:
|
||||
return await self._frames.__anext__()
|
||||
|
||||
async def aclose(self) -> None:
|
||||
await self._frames.aclose()
|
||||
|
||||
def partial_response_on_disconnect(self) -> "ResponseIncompleteEvent | None":
|
||||
return self._wrapper.partial_response_on_disconnect()
|
||||
|
|
|
|||
|
|
@ -464,33 +464,32 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
|
||||
return input_items
|
||||
|
||||
@staticmethod
|
||||
def _translate_tool_to_responses_api(tool_dict: Mapping[str, object]) -> dict[str, object]:
|
||||
tool_type: Final = tool_dict.get("type", "")
|
||||
tool_name: Final = tool_dict.get("name", "")
|
||||
# web_search tool
|
||||
if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
|
||||
return {"type": "web_search_preview"}
|
||||
# Responses turns strict mode on when `strict` is omitted, silently rewriting
|
||||
# `required` to every property. Anthropic tools are non-strict unless asked.
|
||||
func_tool: Final[dict[str, object]] = {
|
||||
"type": "function",
|
||||
"name": tool_name,
|
||||
"strict": bool(tool_dict.get("strict")),
|
||||
}
|
||||
if "description" in tool_dict:
|
||||
func_tool["description"] = tool_dict["description"]
|
||||
if "input_schema" in tool_dict:
|
||||
func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])
|
||||
return func_tool
|
||||
|
||||
def translate_tools_to_responses_api(
|
||||
self,
|
||||
tools: list[AllAnthropicToolsValues],
|
||||
) -> list[dict[str, object]]:
|
||||
"""Convert Anthropic tool definitions to Responses API function tools."""
|
||||
result: Final[list[dict[str, object]]] = []
|
||||
for tool in tools:
|
||||
tool_dict = cast(dict[str, Any], tool)
|
||||
tool_type = tool_dict.get("type", "")
|
||||
tool_name = tool_dict.get("name", "")
|
||||
# web_search tool
|
||||
if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search":
|
||||
result.append({"type": "web_search_preview"})
|
||||
continue
|
||||
# Responses turns strict mode on when `strict` is omitted, silently rewriting
|
||||
# `required` to every property. Anthropic tools are non-strict unless asked.
|
||||
func_tool: dict[str, object] = {
|
||||
"type": "function",
|
||||
"name": tool_name,
|
||||
"strict": bool(tool_dict.get("strict")),
|
||||
}
|
||||
if "description" in tool_dict:
|
||||
func_tool["description"] = tool_dict["description"]
|
||||
if "input_schema" in tool_dict:
|
||||
func_tool["parameters"] = json_copy_or_original(tool_dict["input_schema"])
|
||||
result.append(func_tool)
|
||||
return result
|
||||
return [self._translate_tool_to_responses_api(tool) for tool in tools]
|
||||
|
||||
@staticmethod
|
||||
def translate_tool_choice_to_responses_api(
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import Final
|
|||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from pydantic_core import ErrorDetails
|
||||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
|
|
@ -394,7 +395,9 @@ _MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, ob
|
|||
_LABELS: Final[TypeAdapter[tuple[str, ...]]] = TypeAdapter(tuple[str, ...])
|
||||
|
||||
|
||||
def _normalize_answer(value: object) -> Mapping[str, object]:
|
||||
def _normalize_answer(value: object) -> object:
|
||||
if not isinstance(value, Mapping):
|
||||
return value
|
||||
answer: Final = _MAPPING.validate_python(value)
|
||||
legend: Final = answer.get("legend")
|
||||
if answer.get("type") != "score" or not isinstance(legend, list):
|
||||
|
|
@ -403,11 +406,36 @@ def _normalize_answer(value: object) -> Mapping[str, object]:
|
|||
return {**answer, "legend": {str(index): label for index, label in enumerate(labels)}}
|
||||
|
||||
|
||||
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
|
||||
def _normalize_systemone_response(payload: object) -> object:
|
||||
if not isinstance(payload, Mapping):
|
||||
return payload
|
||||
response: Final = _MAPPING.validate_python(payload)
|
||||
answers: Final = _MAPPING.validate_python(response.get("answers"))
|
||||
normalized: Final = {**response, "answers": {name: _normalize_answer(answer) for name, answer in answers.items()}}
|
||||
return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(normalized), request)
|
||||
answers: Final = response.get("answers")
|
||||
if not isinstance(answers, Mapping):
|
||||
return response
|
||||
return {
|
||||
**response,
|
||||
"answers": {name: _normalize_answer(answer) for name, answer in _MAPPING.validate_python(answers).items()},
|
||||
}
|
||||
|
||||
|
||||
def _validation_detail(detail: ErrorDetails) -> str:
|
||||
location: Final = ".".join(str(part) for part in detail["loc"])
|
||||
return f"{location}: {detail['msg']}" if location else detail["msg"]
|
||||
|
||||
|
||||
def _validation_summary(error: ValidationError) -> str:
|
||||
return "; ".join(_validation_detail(detail) for detail in error.errors(include_url=False, include_input=False))
|
||||
|
||||
|
||||
class DecisionsResponseValidationError(BaseLLMException):
|
||||
pass
|
||||
|
||||
|
||||
def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse:
|
||||
return systemone_response_to_ir(
|
||||
_SYSTEMONE_RESPONSE_ADAPTER.validate_python(_normalize_systemone_response(payload)), request
|
||||
)
|
||||
|
||||
|
||||
def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None:
|
||||
|
|
@ -627,9 +655,10 @@ class BaseDecisionsConfig(ABC):
|
|||
try:
|
||||
return self.parse_response(payload, request)
|
||||
except ValidationError as error:
|
||||
raise BaseLLMException(
|
||||
raise DecisionsResponseValidationError(
|
||||
status_code=500,
|
||||
message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}",
|
||||
message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: "
|
||||
f"{error.title} {_validation_summary(error)}",
|
||||
) from error
|
||||
|
||||
def get_error_class(
|
||||
|
|
|
|||
|
|
@ -185,7 +185,7 @@ class BaseTranslation(ABC):
|
|||
request_data: dict | None = None,
|
||||
stream_transform_sink: StreamTransformSink | None = None,
|
||||
deliver_ended_stream_rewrites: bool = False,
|
||||
) -> Any:
|
||||
) -> object:
|
||||
"""
|
||||
Process output streaming response with guardrails.
|
||||
|
||||
|
|
|
|||
|
|
@ -68,6 +68,16 @@ class HarnessTurnRequest:
|
|||
cwd: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessSessionCopy:
|
||||
"""A saved session under fresh ids: written to `path` in the sandbox, then loaded by `import_command`."""
|
||||
|
||||
session_id: str
|
||||
path: str
|
||||
data: bytes
|
||||
import_command: Sequence[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HarnessTurnResponse:
|
||||
"""What the runtime produced for one turn, after the process exited."""
|
||||
|
|
@ -128,6 +138,18 @@ class BaseCLIHarnessConfig(BaseHarnessConfig[OptionsT], Generic[OptionsT, Stream
|
|||
) -> HarnessTurnRequest:
|
||||
"""argv / env / stdin for one turn. native_session_id is set after the first turn."""
|
||||
|
||||
def transform_resume_export_command(self, native_session_id: str) -> Sequence[str] | None:
|
||||
"""Command that prints a saved session before resuming one this handler did not run.
|
||||
|
||||
None resumes the saved session as is. A non-zero exit or a timeout fails the turn, and the next turn
|
||||
tries again.
|
||||
"""
|
||||
return None
|
||||
|
||||
def transform_resume_copy(self, exported: str, private_dir: str, workdir: str) -> HarnessSessionCopy | None:
|
||||
"""The copy to import and continue, from the export command's stdout, or None to continue the original."""
|
||||
return None
|
||||
|
||||
@abstractmethod
|
||||
def create_stream_state(self) -> StreamStateT:
|
||||
"""Fresh per-turn parser state."""
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import PrismaClient as _PrismaClient
|
||||
from litellm.router import Router as _Router
|
||||
|
||||
Span = _Span | Any
|
||||
Span = _Span
|
||||
InternalUsageCache = _InternalUsageCache
|
||||
PrismaClient = _PrismaClient
|
||||
Router = _Router
|
||||
|
|
@ -550,7 +550,7 @@ class BaseManagedResource(ABC, Generic[ResourceObjectType]):
|
|||
limit: int | None = None,
|
||||
after: str | None = None,
|
||||
additional_filters: Mapping[str, object] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
List resources created by a user.
|
||||
|
||||
|
|
|
|||
|
|
@ -80,6 +80,12 @@ class _AssumeRoleParams(TypedDict):
|
|||
Tags: ReadOnly[NotRequired[tuple[AwsSessionTag, ...]]]
|
||||
|
||||
|
||||
class _StsClientKwargs(TypedDict):
|
||||
verify: ReadOnly[bool | str]
|
||||
endpoint_url: ReadOnly[NotRequired[str]]
|
||||
region_name: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
def _assume_role_params(
|
||||
aws_role_name: str,
|
||||
aws_session_name: str,
|
||||
|
|
@ -275,7 +281,7 @@ class BaseAWSLLM(SignsRequestsWithAWS):
|
|||
"aws_session_tags",
|
||||
]
|
||||
|
||||
def _get_ssl_verify(self, ssl_verify: bool | str | None = None):
|
||||
def _get_ssl_verify(self, ssl_verify: bool | str | None = None) -> bool | str:
|
||||
"""
|
||||
Get SSL verification setting for boto3 clients.
|
||||
|
||||
|
|
@ -875,15 +881,18 @@ class BaseAWSLLM(SignsRequestsWithAWS):
|
|||
aws_sts_endpoint: str | None = None,
|
||||
ssl_verify: bool | str | None = None,
|
||||
aws_region_name: str | None = None,
|
||||
) -> dict:
|
||||
) -> _StsClientKwargs:
|
||||
"""STS client kwargs with aligned endpoint_url and region_name (SigV4)."""
|
||||
kwargs: Final[dict] = {"verify": self._get_ssl_verify(ssl_verify)}
|
||||
if aws_sts_endpoint is not None:
|
||||
kwargs["endpoint_url"] = aws_sts_endpoint
|
||||
sts_region: Final = self._resolve_sts_region(aws_sts_endpoint, aws_region_name)
|
||||
if sts_region is not None:
|
||||
kwargs["region_name"] = sts_region
|
||||
return kwargs
|
||||
verify: Final = self._get_ssl_verify(ssl_verify)
|
||||
match (aws_sts_endpoint, self._resolve_sts_region(aws_sts_endpoint, aws_region_name)):
|
||||
case (None, None):
|
||||
return _StsClientKwargs(verify=verify)
|
||||
case (None, region):
|
||||
return _StsClientKwargs(verify=verify, region_name=region)
|
||||
case (endpoint, None):
|
||||
return _StsClientKwargs(verify=verify, endpoint_url=endpoint)
|
||||
case (endpoint, region):
|
||||
return _StsClientKwargs(verify=verify, endpoint_url=endpoint, region_name=region)
|
||||
|
||||
def get_aws_region_name_for_non_llm_api_calls(
|
||||
self,
|
||||
|
|
@ -1813,13 +1822,13 @@ async def run_aws_signing(
|
|||
)
|
||||
|
||||
|
||||
async def sign_request_off_loop_if_aws(
|
||||
async def run_off_loop_if_aws(
|
||||
provider_config: object,
|
||||
sign_request: Callable[_SignParams, _SignedRequest],
|
||||
call: Callable[_SignParams, _SignedRequest],
|
||||
/,
|
||||
*args: _SignParams.args,
|
||||
**kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped sign_request signature
|
||||
**kwargs: _SignParams.kwargs, # kwargs-ok: ParamSpec forwarding keeps the wrapped call's signature
|
||||
) -> _SignedRequest:
|
||||
if isinstance(provider_config, SignsRequestsWithAWS):
|
||||
return await run_aws_signing(sign_request, *args, **kwargs)
|
||||
return sign_request(*args, **kwargs)
|
||||
return await run_aws_signing(call, *args, **kwargs)
|
||||
return call(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,10 @@ from typing import TYPE_CHECKING, Final, Literal
|
|||
from openai.types.batch import BatchRequestCounts
|
||||
from openai.types.batch import Metadata as OpenAIBatchMetadata
|
||||
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_control_plane_base_url,
|
||||
configured_bedrock_control_plane_endpoint,
|
||||
)
|
||||
from litellm.types.llms.bedrock import AwsAuthParams, AwsSessionTag
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
|
|
@ -156,10 +159,14 @@ class BedrockBatchesHandler:
|
|||
aws_session_tags=aws_session_tags,
|
||||
)
|
||||
creds: Final = BedrockBatchesConfig().resolve_credentials(auth_params, region)
|
||||
control_plane_endpoint: Final = configured_bedrock_control_plane_endpoint(
|
||||
kwargs.get("aws_bedrock_control_plane_endpoint")
|
||||
)
|
||||
|
||||
client: Final = boto3.client(
|
||||
"bedrock",
|
||||
region_name=region,
|
||||
endpoint_url=control_plane_endpoint,
|
||||
aws_access_key_id=creds.access_key,
|
||||
aws_secret_access_key=creds.secret_key,
|
||||
aws_session_token=creds.token,
|
||||
|
|
@ -171,6 +178,7 @@ class BedrockBatchesHandler:
|
|||
batch_id=batch_id,
|
||||
aws_region_name=region,
|
||||
logging_obj=logging_obj,
|
||||
aws_bedrock_control_plane_endpoint=control_plane_endpoint,
|
||||
**auth_params.model_dump(),
|
||||
)
|
||||
|
||||
|
|
@ -270,7 +278,7 @@ class BedrockBatchesHandler:
|
|||
batch_id: str,
|
||||
aws_region_name: str | None = None,
|
||||
logging_obj=None,
|
||||
**kwargs,
|
||||
**kwargs: object,
|
||||
) -> "LiteLLMBatch":
|
||||
"""
|
||||
Handle ``GetModelInvocationJob`` status check for AWS Bedrock bulk batch
|
||||
|
|
@ -294,7 +302,10 @@ class BedrockBatchesHandler:
|
|||
``aws_session_token``, ``aws_profile_name``,
|
||||
``aws_role_name``, ``aws_session_name``,
|
||||
``aws_web_identity_token``, ``aws_sts_endpoint``,
|
||||
``aws_external_id``, ``aws_session_tags``). Unknown keys are ignored.
|
||||
``aws_external_id``, ``aws_session_tags``), plus
|
||||
``aws_bedrock_control_plane_endpoint`` to send the call to a VPC
|
||||
endpoint or gateway instead of the regional ``bedrock`` host.
|
||||
Unknown keys are ignored.
|
||||
|
||||
Returns:
|
||||
``LiteLLMBatch`` shaped like an OpenAI Batch resource.
|
||||
|
|
@ -318,10 +329,14 @@ class BedrockBatchesHandler:
|
|||
from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
|
||||
|
||||
creds: Final = BedrockBatchesConfig().resolve_credentials(AwsAuthParams.model_validate(kwargs), region)
|
||||
control_plane_endpoint: Final = configured_bedrock_control_plane_endpoint(
|
||||
kwargs.get("aws_bedrock_control_plane_endpoint")
|
||||
)
|
||||
|
||||
client: Final = boto3.client(
|
||||
"bedrock",
|
||||
region_name=region,
|
||||
endpoint_url=control_plane_endpoint,
|
||||
aws_access_key_id=creds.access_key,
|
||||
aws_secret_access_key=creds.secret_key,
|
||||
aws_session_token=creds.token,
|
||||
|
|
@ -334,14 +349,13 @@ class BedrockBatchesHandler:
|
|||
# `GetModelInvocationJob` accepts either form, but only the bare id
|
||||
# produces a sensible-looking URL in logs.
|
||||
url_path_id: Final = _extract_job_id_from_arn(batch_id) or batch_id
|
||||
control_plane_base: Final = bedrock_control_plane_base_url(control_plane_endpoint, region)
|
||||
logging_obj.pre_call(
|
||||
input=batch_id,
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": {"jobIdentifier": batch_id},
|
||||
"api_base": (
|
||||
f"https://bedrock.{region}.{get_aws_dns_suffix(region)}/model-invocation-job/{url_path_id}"
|
||||
),
|
||||
"api_base": f"{control_plane_base}/model-invocation-job/{url_path_id}",
|
||||
},
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Final, Literal, cast
|
|||
from httpx import Headers, Response
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix, is_bedrock_arn
|
||||
from litellm.litellm_core_utils.aws_partition import is_bedrock_arn
|
||||
from litellm.litellm_core_utils.cloud_storage_security import (
|
||||
BEDROCK_MANAGED_S3_BATCH_PREFIX,
|
||||
)
|
||||
|
|
@ -32,6 +32,7 @@ from litellm.types.utils import LiteLLMBatch, LlmProviders, Usage
|
|||
from ..base_aws_llm import BaseAWSLLM
|
||||
from ..common_utils import (
|
||||
CommonBatchFilesUtils,
|
||||
bedrock_control_plane_base_url,
|
||||
merge_bedrock_aws_request_params,
|
||||
resolve_s3_bucket_owner,
|
||||
resolve_s3_encryption_key_id,
|
||||
|
|
@ -42,7 +43,7 @@ if TYPE_CHECKING:
|
|||
|
||||
# Bedrock batch input files are uploaded as
|
||||
# s3://bucket/litellm-bedrock-files-{model, ":" -> "-"}-{uuid4}.jsonl (see
|
||||
# BedrockFilesTransformation._get_s3_object_name). A uuid4 is always 36 hex/dash
|
||||
# _batch_s3_object_name in litellm/llms/bedrock/files/transformation.py). A uuid4 is always 36 hex/dash
|
||||
# characters, so it can be stripped off the end unambiguously even though the
|
||||
# model name itself may contain dashes.
|
||||
_S3_BATCH_FILE_UUID_SUFFIX_PATTERN: Final = re.compile(
|
||||
|
|
@ -177,11 +178,10 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
|
|||
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
|
||||
aws_region_name: Final = self._get_aws_region_name(request_params, model)
|
||||
|
||||
# Bedrock model invocation job endpoint
|
||||
# Format: https://bedrock.{region}.{partition dns suffix}/model-invocation-job
|
||||
bedrock_endpoint: Final = (
|
||||
f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/model-invocation-job"
|
||||
control_plane_base: Final = bedrock_control_plane_base_url(
|
||||
request_params.get("aws_bedrock_control_plane_endpoint"), aws_region_name
|
||||
)
|
||||
bedrock_endpoint: Final = f"{control_plane_base}/model-invocation-job"
|
||||
|
||||
return bedrock_endpoint
|
||||
|
||||
|
|
@ -279,9 +279,10 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
|
|||
# Use common utility for AWS signing
|
||||
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
|
||||
aws_region_name: Final = self._get_aws_region_name(request_params, model)
|
||||
endpoint_url: Final = (
|
||||
f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}/model-invocation-job"
|
||||
control_plane_base: Final = bedrock_control_plane_base_url(
|
||||
request_params.get("aws_bedrock_control_plane_endpoint"), aws_region_name
|
||||
)
|
||||
endpoint_url: Final = f"{control_plane_base}/model-invocation-job"
|
||||
signed_headers, signed_data = self.common_utils.sign_aws_request(
|
||||
service_name="bedrock",
|
||||
data=bedrock_request,
|
||||
|
|
@ -431,12 +432,13 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
|
|||
import urllib.parse as _ul
|
||||
|
||||
encoded_arn: Final = _ul.quote(batch_id, safe="")
|
||||
endpoint_url: Final = (
|
||||
f"https://bedrock.{region}.{get_aws_dns_suffix(region)}/model-invocation-job/{encoded_arn}"
|
||||
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
|
||||
control_plane_base: Final = bedrock_control_plane_base_url(
|
||||
request_params.get("aws_bedrock_control_plane_endpoint"), region
|
||||
)
|
||||
endpoint_url: Final = f"{control_plane_base}/model-invocation-job/{encoded_arn}"
|
||||
|
||||
# Use common utility for AWS signing
|
||||
request_params: Final = merge_bedrock_aws_request_params(litellm_params, optional_params)
|
||||
signed_headers, _ = self.common_utils.sign_aws_request(
|
||||
service_name="bedrock",
|
||||
data={}, # GET request has no body
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ def _response_header(headers: Mapping[str, str], name: str) -> str | None:
|
|||
return headers.get(name, None)
|
||||
|
||||
|
||||
def _bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
|
||||
def bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
|
||||
try:
|
||||
guardrail_config: Final = _GUARDRAIL_CONFIG_VALIDATOR.validate_python(raw_guardrail_config)
|
||||
except ValidationError as e:
|
||||
|
|
@ -440,7 +440,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
|
|||
guardrail_headers: Final = (
|
||||
()
|
||||
if raw_guardrail_config is None
|
||||
else tuple(_bedrock_invoke_guardrail_headers(raw_guardrail_config).items())
|
||||
else tuple(bedrock_invoke_guardrail_headers(raw_guardrail_config).items())
|
||||
)
|
||||
owned_names, metadata_headers = bedrock_request_metadata_headers(litellm_params)
|
||||
return merge_bedrock_invoke_headers(headers, guardrail_headers, metadata_headers, owned_names)
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
DEFAULT_ANTHROPIC_API_VERSION,
|
||||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.bedrock.base_aws_llm import run_aws_signing
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
|
@ -22,14 +23,14 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
|
|||
|
||||
def validate_anthropic_messages_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict[str, str], # mutable-ok: overrides the base signature
|
||||
model: str,
|
||||
messages: list[Any],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
messages: list[object], # mutable-ok: overrides the base signature
|
||||
optional_params: dict[str, object], # mutable-ok: overrides the base signature
|
||||
litellm_params: dict[str, object], # mutable-ok: overrides the base signature
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[dict, str | None]:
|
||||
) -> tuple[dict[str, str], str | None]: # mutable-ok: overrides the base signature
|
||||
workspace_id: Final = self._get_workspace_id(optional_params, litellm_params)
|
||||
if workspace_id is None:
|
||||
raise litellm.AuthenticationError(
|
||||
|
|
@ -66,6 +67,27 @@ class BedrockClaudePlatformMessagesConfig(BedrockClaudePlatformMixin, AnthropicM
|
|||
api_base,
|
||||
)
|
||||
|
||||
async def avalidate_anthropic_messages_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: mirrors validate_anthropic_messages_environment
|
||||
model: str,
|
||||
messages: list[object], # mutable-ok: mirrors validate_anthropic_messages_environment
|
||||
optional_params: dict[str, object], # mutable-ok: mirrors validate_anthropic_messages_environment
|
||||
litellm_params: dict[str, object], # mutable-ok: mirrors validate_anthropic_messages_environment
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> tuple[dict[str, str], str | None]: # mutable-ok: mirrors validate_anthropic_messages_environment
|
||||
return await run_aws_signing(
|
||||
self.validate_anthropic_messages_environment,
|
||||
headers=headers,
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params=litellm_params,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
)
|
||||
|
||||
def transform_anthropic_messages_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
|
|||
|
|
@ -138,6 +138,25 @@ def merge_bedrock_aws_request_params(
|
|||
return request_params
|
||||
|
||||
|
||||
def configured_bedrock_control_plane_endpoint(endpoint: object) -> str | None:
|
||||
"""An ``aws_bedrock_control_plane_endpoint`` value without a trailing slash, or None when unset."""
|
||||
if isinstance(endpoint, str) and endpoint.strip():
|
||||
return endpoint.strip().rstrip("/")
|
||||
return None
|
||||
|
||||
|
||||
def bedrock_control_plane_base_url(endpoint: object, aws_region_name: str) -> str:
|
||||
"""Base URL for Bedrock control-plane calls such as model invocation jobs.
|
||||
|
||||
A configured ``aws_bedrock_control_plane_endpoint`` (a VPC endpoint or a compatible gateway)
|
||||
wins over the regional ``bedrock`` host. ``api_base`` is deliberately not used here, since on a
|
||||
Bedrock deployment it points at the runtime host, which does not serve these APIs.
|
||||
"""
|
||||
return configured_bedrock_control_plane_endpoint(endpoint) or (
|
||||
f"https://bedrock.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
|
||||
)
|
||||
|
||||
|
||||
def s3_static_key_pair(params: Mapping[str, object]) -> tuple[str, str] | None:
|
||||
"""The s3_access_key_id / s3_secret_access_key pair when both are set, otherwise None."""
|
||||
s3_access_key_id: Final = params.get("s3_access_key_id")
|
||||
|
|
|
|||
|
|
@ -48,7 +48,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
verbose_logger.debug("Processing CountTokens request for resolved model: %s", resolved_model)
|
||||
|
||||
# Get AWS region using existing LiteLLM function
|
||||
aws_region_name: Final = self.get_aws_region_name(
|
||||
aws_region_name: Final = await run_aws_signing(
|
||||
self.get_aws_region_name,
|
||||
optional_params=litellm_params,
|
||||
model=resolved_model,
|
||||
model_id=None,
|
||||
|
|
@ -64,7 +65,8 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
|
|||
# Get endpoint URL using simplified function
|
||||
api_base: Final = litellm_params.get("api_base", None)
|
||||
aws_bedrock_runtime_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint", None)
|
||||
endpoint_url: Final = self.get_bedrock_count_tokens_endpoint(
|
||||
endpoint_url: Final = await run_aws_signing(
|
||||
self.get_bedrock_count_tokens_endpoint,
|
||||
model=resolved_model,
|
||||
aws_region_name=aws_region_name,
|
||||
api_base=api_base,
|
||||
|
|
|
|||
|
|
@ -54,10 +54,10 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
|
|||
) -> dict[str, JsonValue]:
|
||||
try:
|
||||
request: Final = _COUNT_REQUEST.validate_python(request_data)
|
||||
aws_region_name: Final = self.get_aws_region_name(
|
||||
optional_params=litellm_params, model=resolved_model, model_id=None
|
||||
aws_region_name: Final = await run_aws_signing(
|
||||
self.get_aws_region_name, optional_params=litellm_params, model=resolved_model, model_id=None
|
||||
)
|
||||
endpoint_url: Final = self.get_mantle_count_tokens_endpoint(aws_region_name)
|
||||
endpoint_url: Final = await run_aws_signing(self.get_mantle_count_tokens_endpoint, aws_region_name)
|
||||
body: Final = self._anthropic_config.transform_request_to_count_tokens(
|
||||
model=resolved_model,
|
||||
messages=request["messages"],
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import os
|
|||
import posixpath
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Iterable, Mapping, MutableMapping, Sequence
|
||||
from collections.abc import Iterable, Iterator, Mapping, MutableMapping, Sequence
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
|
|
@ -23,6 +23,7 @@ from typing_extensions import ReadOnly
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.files.utils import FilesAPIUtils
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.cloud_storage_security import (
|
||||
|
|
@ -324,7 +325,10 @@ _REJECTED_FILE_ID_REQUEST_URL: Final = "https://litellm.ai"
|
|||
|
||||
|
||||
def _rejected_file_id(reason: ValueError) -> BedrockError:
|
||||
message: Final = str(reason)
|
||||
return _client_error(str(reason))
|
||||
|
||||
|
||||
def _client_error(message: str) -> BedrockError:
|
||||
return BedrockError(
|
||||
status_code=400,
|
||||
message=message,
|
||||
|
|
@ -336,6 +340,92 @@ def _rejected_file_id(reason: ValueError) -> BedrockError:
|
|||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _BatchInputLine:
|
||||
line_number: int
|
||||
record: _OpenAIBatchRecord
|
||||
|
||||
|
||||
_BATCH_RECORD_CONVERSION_ERRORS: Final = (AttributeError, LookupError, TypeError, ValueError, BadRequestError)
|
||||
|
||||
|
||||
def _batch_line_error(line_number: int, problem: str) -> BedrockError:
|
||||
return _client_error(f"Batch input file line {line_number} {problem}")
|
||||
|
||||
|
||||
def _decode_batch_input(content: str | bytes) -> str:
|
||||
if isinstance(content, str):
|
||||
return content.removeprefix("\ufeff")
|
||||
try:
|
||||
return content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError as e:
|
||||
raise _batch_line_error(content.count(b"\n", 0, e.start) + 1, f"is not valid UTF-8: {e.reason}") from e
|
||||
|
||||
|
||||
def _load_batch_line(line_number: int, line: str) -> _OpenAIBatchRecord:
|
||||
try:
|
||||
return json.loads(line)
|
||||
except json.JSONDecodeError as e:
|
||||
raise _batch_line_error(line_number, f"is not valid JSON: {e.msg} at column {e.colno}") from e
|
||||
except (ValueError, RecursionError) as e:
|
||||
raise _batch_line_error(line_number, f"could not be parsed: {e}") from e
|
||||
|
||||
|
||||
def _json_object(value: object) -> Mapping[str, object] | None:
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def _batch_json_object(line_number: int, line: str) -> _OpenAIBatchRecord:
|
||||
record: Final = _load_batch_line(line_number, line)
|
||||
if _json_object(record) is None:
|
||||
raise _batch_line_error(line_number, "is not a JSON object")
|
||||
return record
|
||||
|
||||
|
||||
def _batch_body_problem(record: Mapping[str, object]) -> str | None:
|
||||
body: Final = _json_object(record.get("body"))
|
||||
if body is None:
|
||||
return "has a body that is missing or not a JSON object"
|
||||
if "model" in body and not isinstance(body["model"], str):
|
||||
return "has a body.model that is not a string"
|
||||
return None
|
||||
|
||||
|
||||
def _batch_input_line(line_number: int, line: str) -> _BatchInputLine:
|
||||
record: Final = _batch_json_object(line_number, line)
|
||||
problem: Final = _batch_body_problem(record)
|
||||
if problem is not None:
|
||||
raise _batch_line_error(line_number, problem)
|
||||
return _BatchInputLine(line_number=line_number, record=record)
|
||||
|
||||
|
||||
def _non_blank_lines(file_content: str) -> Iterator[tuple[int, str]]:
|
||||
numbered_lines: Final = enumerate(file_content.splitlines(), start=1)
|
||||
return ((line_number, line) for line_number, line in numbered_lines if line.strip())
|
||||
|
||||
|
||||
def _parse_batch_jsonl(file_content: str) -> tuple[_BatchInputLine, ...]:
|
||||
return tuple(_batch_input_line(line_number, line) for line_number, line in _non_blank_lines(file_content))
|
||||
|
||||
|
||||
def _batch_file_model(file_content: str) -> str | None:
|
||||
records: Final = tuple(
|
||||
_batch_json_object(line_number, line) for line_number, line in _non_blank_lines(file_content)
|
||||
)
|
||||
if not records:
|
||||
return None
|
||||
body: Final = _json_object(records[0].get("body"))
|
||||
model: Final = body.get("model") if body is not None else None
|
||||
return model if isinstance(model, str) else ""
|
||||
|
||||
|
||||
def _batch_s3_object_name(model: str) -> str:
|
||||
safe_model: Final = sanitize_cloud_object_component(
|
||||
model.removeprefix("bedrock/").replace(":", "-"), fallback="model"
|
||||
)
|
||||
return f"{BEDROCK_MANAGED_S3_BATCH_PREFIX}{safe_model}-{uuid.uuid4()}.jsonl"
|
||||
|
||||
|
||||
def _resolve_managed_s3_object(file_id: str, litellm_params: Mapping[str, object]) -> tuple[str, str]:
|
||||
configured_bucket_names: Final = get_configured_s3_bucket_names(litellm_params)
|
||||
allow_legacy_cloud_file_ids: Final = should_allow_legacy_cloud_file_ids(litellm_params)
|
||||
|
|
@ -545,29 +635,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
# File-like objects need to be read
|
||||
content = file_content.read()
|
||||
|
||||
# Ensure content is string
|
||||
if isinstance(content, bytes):
|
||||
content = content.decode("utf-8")
|
||||
|
||||
return content
|
||||
|
||||
def _get_s3_object_name_from_batch_jsonl(
|
||||
self,
|
||||
openai_jsonl_content: Sequence[_OpenAIBatchRecord],
|
||||
) -> str:
|
||||
"""
|
||||
Gets a unique S3 object name for the Bedrock batch processing job
|
||||
|
||||
named as: litellm-bedrock-files/{model}/{uuid}
|
||||
"""
|
||||
_model = openai_jsonl_content[0].get("body", {}).get("model", "")
|
||||
# Remove bedrock/ prefix if present
|
||||
_model = _model.removeprefix("bedrock/")
|
||||
|
||||
safe_model: Final = sanitize_cloud_object_component(_model.replace(":", "-"), fallback="model")
|
||||
|
||||
object_name: Final = f"{BEDROCK_MANAGED_S3_BATCH_PREFIX}{safe_model}-{uuid.uuid4()}.jsonl"
|
||||
return object_name
|
||||
return _decode_batch_input(content)
|
||||
|
||||
def get_object_name(self, extracted_file_data: ExtractedFileData, purpose: str) -> str:
|
||||
"""
|
||||
|
|
@ -581,11 +649,9 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
if purpose == "batch":
|
||||
## 1. If jsonl, check if there's a model name
|
||||
file_content: Final = self._get_content_from_openai_file(extracted_file_data_content)
|
||||
|
||||
# Split into lines and parse each line as JSON
|
||||
openai_jsonl_content: Final = [json.loads(line) for line in file_content.splitlines() if line.strip()]
|
||||
if len(openai_jsonl_content) > 0:
|
||||
return self._get_s3_object_name_from_batch_jsonl(openai_jsonl_content)
|
||||
batch_model: Final = _batch_file_model(file_content)
|
||||
if batch_model is not None:
|
||||
return _batch_s3_object_name(batch_model)
|
||||
|
||||
## 2. If not jsonl, store under a server-generated managed object name
|
||||
filename: Final = extracted_file_data.get("filename")
|
||||
|
|
@ -1080,42 +1146,60 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
}
|
||||
}
|
||||
"""
|
||||
return [
|
||||
self._transform_openai_batch_record(idx, openai_record, target_model)
|
||||
for idx, openai_record in enumerate(openai_jsonl_content)
|
||||
]
|
||||
|
||||
def transform_batch_input_lines(
|
||||
self, batch_input_lines: Sequence[_BatchInputLine], target_model: str = ""
|
||||
) -> tuple[_BedrockBatchRecord, ...]:
|
||||
return tuple(
|
||||
self._transform_batch_input_line(idx, batch_input_line, target_model)
|
||||
for idx, batch_input_line in enumerate(batch_input_lines)
|
||||
)
|
||||
|
||||
def _transform_batch_input_line(
|
||||
self, idx: int, batch_input_line: _BatchInputLine, target_model: str
|
||||
) -> _BedrockBatchRecord:
|
||||
try:
|
||||
return self._transform_openai_batch_record(idx, batch_input_line.record, target_model)
|
||||
except _BATCH_RECORD_CONVERSION_ERRORS as e:
|
||||
raise _batch_line_error(
|
||||
batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}"
|
||||
) from e
|
||||
|
||||
def _transform_openai_batch_record(
|
||||
self, idx: int, openai_record: _OpenAIBatchRecord, target_model: str
|
||||
) -> _BedrockBatchRecord:
|
||||
import litellm
|
||||
|
||||
bedrock_jsonl_content: Final = []
|
||||
for idx, _openai_jsonl_content in enumerate(openai_jsonl_content):
|
||||
# Extract the request body from OpenAI format
|
||||
openai_body = _openai_jsonl_content.get("body", {})
|
||||
record_model = openai_body.get("model", "")
|
||||
resolved_model = litellm.model_alias_map.get(record_model, record_model)
|
||||
model_for_transform, provider = self._resolve_batch_record_model_and_provider(
|
||||
record_model=resolved_model, target_model=target_model
|
||||
openai_body: Final = openai_record.get("body", {})
|
||||
record_model: Final = openai_body.get("model", "")
|
||||
resolved_model: Final = litellm.model_alias_map.get(record_model, record_model)
|
||||
model_for_transform, provider = self._resolve_batch_record_model_and_provider(
|
||||
record_model=resolved_model, target_model=target_model
|
||||
)
|
||||
|
||||
# Route to the embedding transformer when the OpenAI batch line
|
||||
# targets /v1/embeddings; every other endpoint shape is normalized
|
||||
# to chat completions first. We branch here (rather than inside
|
||||
# `_map_openai_to_bedrock_params`) so the chat helper keeps its
|
||||
# narrow contract and the embedding helper can evolve independently.
|
||||
record_kind: Final = self._classify_batch_record(openai_record)
|
||||
model_input: Final = (
|
||||
self._map_openai_embedding_to_bedrock_params(openai_request_body=openai_body, model=model_for_transform)
|
||||
if record_kind is BedrockBatchRecordKind.EMBEDDING
|
||||
else self._map_openai_to_bedrock_params(
|
||||
openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind),
|
||||
model=model_for_transform,
|
||||
provider=provider,
|
||||
)
|
||||
|
||||
# Route to the embedding transformer when the OpenAI batch line
|
||||
# targets /v1/embeddings; every other endpoint shape is normalized
|
||||
# to chat completions first. We branch here (rather than inside
|
||||
# `_map_openai_to_bedrock_params`) so the chat helper keeps its
|
||||
# narrow contract and the embedding helper can evolve independently.
|
||||
record_kind = self._classify_batch_record(_openai_jsonl_content)
|
||||
if record_kind is BedrockBatchRecordKind.EMBEDDING:
|
||||
model_input = self._map_openai_embedding_to_bedrock_params(
|
||||
openai_request_body=openai_body, model=model_for_transform
|
||||
)
|
||||
else:
|
||||
model_input = self._map_openai_to_bedrock_params(
|
||||
openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind),
|
||||
model=model_for_transform,
|
||||
provider=provider,
|
||||
)
|
||||
|
||||
# Create Bedrock batch record
|
||||
record_id = _openai_jsonl_content.get("custom_id", f"CALL{str(idx).zfill(7)}")
|
||||
bedrock_record = {"recordId": record_id, "modelInput": model_input}
|
||||
|
||||
bedrock_jsonl_content.append(bedrock_record)
|
||||
return bedrock_jsonl_content
|
||||
)
|
||||
return _BedrockBatchRecord(
|
||||
recordId=openai_record.get("custom_id", f"CALL{str(idx).zfill(7)}"),
|
||||
modelInput=model_input,
|
||||
)
|
||||
|
||||
def transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
self,
|
||||
|
|
@ -1151,11 +1235,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
|
|||
):
|
||||
## Transform JSONL content to Bedrock format
|
||||
original_file_content: Final = self._get_content_from_openai_file(extracted_file_data_content)
|
||||
openai_jsonl_content = [json.loads(line) for line in original_file_content.splitlines() if line.strip()]
|
||||
batch_input_lines: Final = _parse_batch_jsonl(original_file_content)
|
||||
litellm_params_model: Final = litellm_params.get("model")
|
||||
target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "")
|
||||
bedrock_jsonl_content = self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
openai_jsonl_content, target_model=target_model
|
||||
bedrock_jsonl_content: Final = self.transform_batch_input_lines(
|
||||
batch_input_lines, target_model=target_model
|
||||
)
|
||||
file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
|
||||
elif isinstance(extracted_file_data_content, bytes):
|
||||
|
|
@ -1662,37 +1746,11 @@ class BedrockJsonlFilesTransformation:
|
|||
# Read the content of the file
|
||||
file_content: Final = self._get_content_from_openai_file(openai_file_content)
|
||||
|
||||
# Split into lines and parse each line as JSON
|
||||
openai_jsonl_content: Final = [json.loads(line) for line in file_content.splitlines() if line.strip()]
|
||||
bedrock_jsonl_content = self._transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content)
|
||||
batch_input_lines: Final = _parse_batch_jsonl(file_content)
|
||||
bedrock_jsonl_content: Final = BedrockFilesConfig().transform_batch_input_lines(batch_input_lines)
|
||||
bedrock_jsonl_string: Final = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
|
||||
object_name: Final = self._get_s3_object_name(openai_jsonl_content=openai_jsonl_content)
|
||||
return bedrock_jsonl_string, object_name
|
||||
|
||||
def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
|
||||
self, openai_jsonl_content: Sequence[_OpenAIBatchRecord]
|
||||
):
|
||||
"""
|
||||
Delegate to the main BedrockFilesConfig transformation method
|
||||
"""
|
||||
config: Final = BedrockFilesConfig()
|
||||
return config.transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content)
|
||||
|
||||
def _get_s3_object_name(
|
||||
self,
|
||||
openai_jsonl_content: Sequence[_OpenAIBatchRecord],
|
||||
) -> str:
|
||||
"""
|
||||
Gets a unique S3 object name for the Bedrock batch processing job
|
||||
|
||||
named as: litellm-bedrock-files-{model}-{uuid}
|
||||
"""
|
||||
_model = openai_jsonl_content[0].get("body", {}).get("model", "")
|
||||
# Remove bedrock/ prefix if present
|
||||
_model = _model.removeprefix("bedrock/")
|
||||
safe_model: Final = sanitize_cloud_object_component(_model.replace(":", "-"), fallback="model")
|
||||
object_name: Final = f"{BEDROCK_MANAGED_S3_BATCH_PREFIX}{safe_model}-{uuid.uuid4()}.jsonl"
|
||||
return object_name
|
||||
first_model: Final = batch_input_lines[0].record.get("body", {}).get("model", "") if batch_input_lines else ""
|
||||
return bedrock_jsonl_string, _batch_s3_object_name(first_model)
|
||||
|
||||
def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
|
||||
"""
|
||||
|
|
@ -1725,11 +1783,7 @@ class BedrockJsonlFilesTransformation:
|
|||
# File-like objects need to be read
|
||||
content = file_content.read()
|
||||
|
||||
# Ensure content is string
|
||||
if isinstance(content, bytes):
|
||||
content = content.decode("utf-8")
|
||||
|
||||
return content
|
||||
return _decode_batch_input(content)
|
||||
|
||||
def transform_s3_bucket_response_to_openai_file_object(
|
||||
self, create_file_data: CreateFileRequest, s3_upload_response: _S3UploadResponse
|
||||
|
|
|
|||
|
|
@ -14,6 +14,9 @@ from litellm.constants import AWS_ERROR_TYPE_HEADER
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig, PassthroughStreamCollector
|
||||
from litellm.llms.bedrock.chat.converse_to_openai import as_json_array, as_json_object
|
||||
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
||||
bedrock_invoke_guardrail_headers,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionSystemMessage, ChatCompletionUserMessage
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
|
|
@ -42,6 +45,9 @@ if TYPE_CHECKING:
|
|||
BEDROCK_STREAMING_ACTIONS: Final = frozenset({"invoke-with-response-stream", "converse-stream"})
|
||||
_BEDROCK_CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
|
||||
_BEDROCK_INVOKE_ACTIONS: Final = frozenset({"invoke", "invoke-with-response-stream"})
|
||||
_BEDROCK_INVOKE_GUARDRAIL_HEADERS: Final = frozenset(
|
||||
{"x-amzn-bedrock-guardrailidentifier", "x-amzn-bedrock-guardrailversion", "x-amzn-bedrock-trace"}
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_endpoint_action(endpoint: str) -> str:
|
||||
|
|
@ -123,6 +129,31 @@ def _with_invoke_request_metadata(
|
|||
return merge_bedrock_invoke_headers(dict(headers), (), metadata_headers, owned_names), body
|
||||
|
||||
|
||||
def _converse_guardrail_config(action: str, guardrail_config: object) -> object:
|
||||
fields: Final = as_json_object(guardrail_config)
|
||||
if action == "converse-stream" or fields is None:
|
||||
return guardrail_config
|
||||
return {key: value for key, value in fields.items() if key != "streamProcessingMode"}
|
||||
|
||||
|
||||
def _with_deployment_guardrail(
|
||||
action: str,
|
||||
headers: Mapping[str, str],
|
||||
body: Mapping[str, object],
|
||||
guardrail_config: object,
|
||||
) -> tuple[Mapping[str, str], Mapping[str, object]]:
|
||||
if guardrail_config is None:
|
||||
return headers, body
|
||||
if action in _BEDROCK_CONVERSE_ACTIONS:
|
||||
return headers, {**body, "guardrailConfig": _converse_guardrail_config(action, guardrail_config)}
|
||||
if action in _BEDROCK_INVOKE_ACTIONS:
|
||||
guardrail_headers: Final = tuple(bedrock_invoke_guardrail_headers(guardrail_config).items())
|
||||
return merge_bedrock_invoke_headers(
|
||||
dict(headers), (), guardrail_headers, _BEDROCK_INVOKE_GUARDRAIL_HEADERS
|
||||
), body
|
||||
return headers, body
|
||||
|
||||
|
||||
def _is_cache_point(entry: object) -> bool:
|
||||
fields: Final = as_json_object(entry)
|
||||
return fields is not None and "cachePoint" in fields
|
||||
|
|
@ -453,11 +484,14 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
|
|||
body=_converse_body_to_send(body) if action in _BEDROCK_CONVERSE_ACTIONS else body,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
guarded_headers, guarded_body = _with_deployment_guardrail(
|
||||
action, signed_headers, signed_body, litellm_params.get("guardrailConfig")
|
||||
)
|
||||
return self._sign_request(
|
||||
service_name="bedrock",
|
||||
headers=dict(signed_headers),
|
||||
headers=dict(guarded_headers),
|
||||
optional_params=optional_params,
|
||||
request_data=dict(signed_body),
|
||||
request_data=dict(guarded_body),
|
||||
api_base=api_base,
|
||||
model=model,
|
||||
api_key=self.get_bedrock_bearer_token(optional_params),
|
||||
|
|
|
|||
|
|
@ -337,10 +337,7 @@ class BedrockRealtime(BaseAWSLLM):
|
|||
|
||||
# Get AWS region
|
||||
if aws_region_name is None:
|
||||
optional_params: Final = {
|
||||
"aws_region_name": aws_region_name,
|
||||
}
|
||||
aws_region_name = self._get_aws_region_name(optional_params, model)
|
||||
aws_region_name = await run_aws_signing(self.get_aws_region_name, {}, model)
|
||||
|
||||
# Get endpoint URL
|
||||
if api_base is not None:
|
||||
|
|
|
|||
|
|
@ -256,12 +256,11 @@ def merge_bedrock_invoke_headers(
|
|||
"""
|
||||
Merge the ``X-Amzn-*`` headers the Invoke paths derive from params.
|
||||
|
||||
``caller_owned`` (the guardrail headers) defers to a header the caller already set, which is
|
||||
the long-standing behaviour for those. ``proxy_owned_names`` are dropped from the caller's
|
||||
headers unconditionally and re-supplied only from ``proxy_owned``, because those names carry
|
||||
proxy-authenticated identity into an AWS billing record that the caller must not be able to
|
||||
write. Names are compared case-insensitively so a caller cannot leave a second spelling in
|
||||
the dict and let the transport pick the winner.
|
||||
``caller_owned`` defers to a header the caller already set. ``proxy_owned_names`` are dropped
|
||||
from the caller's headers unconditionally and re-supplied only from ``proxy_owned``, for names
|
||||
the caller must not be able to write, such as proxy-authenticated identity headed for an AWS
|
||||
billing record or a deployment's guardrail. Names are compared case-insensitively so a caller
|
||||
cannot leave a second spelling in the dict and let the transport pick the winner.
|
||||
"""
|
||||
if not caller_owned and not proxy_owned and not proxy_owned_names:
|
||||
return headers
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ class ChatGPTConfig(OpenAIConfig):
|
|||
default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id)
|
||||
return {**default_headers, **validated_headers}
|
||||
|
||||
def post_stream_processing(self, stream: Any) -> Any:
|
||||
def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer:
|
||||
return ChatGPTToolCallNormalizer(stream)
|
||||
|
||||
def map_openai_params(
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ from litellm.llms.base_llm.vector_store_files.transformation import (
|
|||
BaseVectorStoreFilesConfig,
|
||||
)
|
||||
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
|
||||
from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS, run_aws_signing, sign_request_off_loop_if_aws
|
||||
from litellm.llms.bedrock.base_aws_llm import SignsRequestsWithAWS, run_aws_signing, run_off_loop_if_aws
|
||||
from litellm.llms.custom_httpx.container_handler import raise_for_error_status
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
|
|
@ -2085,14 +2085,18 @@ class BaseLLMHTTPHandler:
|
|||
raise ValueError(f"No provider config found for provider: {custom_llm_provider}")
|
||||
|
||||
# Validate environment and get headers
|
||||
headers = provider_config.validate_environment(
|
||||
headers = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.validate_environment,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
headers=headers or {},
|
||||
)
|
||||
|
||||
# Transform the request first
|
||||
data: Final = provider_config.transform_search_request(
|
||||
data: Final = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.transform_search_request,
|
||||
query=query,
|
||||
optional_params=optional_params,
|
||||
api_key=api_key,
|
||||
|
|
@ -2108,7 +2112,7 @@ class BaseLLMHTTPHandler:
|
|||
api_key=api_key,
|
||||
)
|
||||
|
||||
signed_headers, signed_json_body = await sign_request_off_loop_if_aws(
|
||||
signed_headers, signed_json_body = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.sign_request,
|
||||
headers=headers,
|
||||
|
|
@ -2236,7 +2240,7 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
provider_config.transform_anthropic_messages_request_on_http_error(e=e, request_data=request_body)
|
||||
recovered_errors = (*recovered_errors, e)
|
||||
headers, signed_json_body = await sign_request_off_loop_if_aws(
|
||||
headers, signed_json_body = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.sign_request,
|
||||
headers=headers,
|
||||
|
|
@ -2405,7 +2409,9 @@ class BaseLLMHTTPHandler:
|
|||
logging_obj.model_call_details.update(request_body)
|
||||
|
||||
# Make the request
|
||||
request_url: Final = anthropic_messages_provider_config.get_complete_url(
|
||||
request_url: Final = await run_off_loop_if_aws(
|
||||
anthropic_messages_provider_config,
|
||||
anthropic_messages_provider_config.get_complete_url,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
|
|
@ -2416,7 +2422,7 @@ class BaseLLMHTTPHandler:
|
|||
stream=stream,
|
||||
)
|
||||
|
||||
headers, signed_json_body = await sign_request_off_loop_if_aws(
|
||||
headers, signed_json_body = await run_off_loop_if_aws(
|
||||
anthropic_messages_provider_config,
|
||||
anthropic_messages_provider_config.sign_request,
|
||||
headers=headers,
|
||||
|
|
@ -2967,7 +2973,9 @@ class BaseLLMHTTPHandler:
|
|||
stream = response_api_optional_request_params.get("stream", False)
|
||||
caller_requested_stream: Final = bool(stream or (extra_body or {}).get("stream"))
|
||||
|
||||
api_base: Final = responses_api_provider_config.get_complete_url(
|
||||
api_base: Final = await run_off_loop_if_aws(
|
||||
responses_api_provider_config,
|
||||
responses_api_provider_config.get_complete_url,
|
||||
api_base=litellm_params.api_base,
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
|
|
@ -3009,7 +3017,7 @@ class BaseLLMHTTPHandler:
|
|||
fake_stream=fake_stream,
|
||||
)
|
||||
|
||||
headers, signed_body = await sign_request_off_loop_if_aws(
|
||||
headers, signed_body = await run_off_loop_if_aws(
|
||||
responses_api_provider_config,
|
||||
responses_api_provider_config.sign_request,
|
||||
headers=validated_headers,
|
||||
|
|
@ -3856,7 +3864,9 @@ class BaseLLMHTTPHandler:
|
|||
litellm_params=litellm_params,
|
||||
api_key=api_key,
|
||||
)
|
||||
complete_api_base: Final = provider_config.get_complete_file_url(
|
||||
complete_api_base: Final = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.get_complete_file_url,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model="",
|
||||
|
|
@ -3867,7 +3877,9 @@ class BaseLLMHTTPHandler:
|
|||
if not complete_api_base:
|
||||
raise ValueError("api_base is required for create_file")
|
||||
return await self.async_create_file(
|
||||
transformed_request=provider_config.transform_create_file_request(
|
||||
transformed_request=await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.transform_create_file_request,
|
||||
model="",
|
||||
create_file_data=create_file_data,
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -4334,7 +4346,9 @@ class BaseLLMHTTPHandler:
|
|||
litellm_params=litellm_params,
|
||||
api_key=api_key,
|
||||
)
|
||||
complete_api_base: Final = provider_config.get_complete_batch_url(
|
||||
complete_api_base: Final = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.get_complete_batch_url,
|
||||
api_base=api_base,
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
|
|
@ -4345,7 +4359,9 @@ class BaseLLMHTTPHandler:
|
|||
if not complete_api_base:
|
||||
raise ValueError("api_base is required for create_batch")
|
||||
return await self.async_create_batch(
|
||||
transformed_request=provider_config.transform_create_batch_request(
|
||||
transformed_request=await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.transform_create_batch_request,
|
||||
model=model,
|
||||
create_batch_data=create_batch_data,
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -4818,7 +4834,9 @@ class BaseLLMHTTPHandler:
|
|||
**(extra_headers or {}),
|
||||
}
|
||||
|
||||
api_base: Final = responses_api_provider_config.get_complete_url(
|
||||
api_base: Final = await run_off_loop_if_aws(
|
||||
responses_api_provider_config,
|
||||
responses_api_provider_config.get_complete_url,
|
||||
api_base=litellm_params.api_base,
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
|
|
@ -4836,7 +4854,7 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
|
||||
|
||||
headers, signed_body = await sign_request_off_loop_if_aws(
|
||||
headers, signed_body = await run_off_loop_if_aws(
|
||||
responses_api_provider_config,
|
||||
responses_api_provider_config.sign_request,
|
||||
headers=validated_headers,
|
||||
|
|
@ -4962,7 +4980,9 @@ class BaseLLMHTTPHandler:
|
|||
async_httpx_client = client
|
||||
|
||||
# Get URL and params from provider config
|
||||
url, params = provider_config.transform_retrieve_file_request(
|
||||
url, params = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.transform_retrieve_file_request,
|
||||
file_id=file_id,
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -5089,7 +5109,9 @@ class BaseLLMHTTPHandler:
|
|||
async_httpx_client = client
|
||||
|
||||
# Get URL and params from provider config
|
||||
url, params = provider_config.transform_delete_file_request(
|
||||
url, params = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.transform_delete_file_request,
|
||||
file_id=file_id,
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -5214,7 +5236,9 @@ class BaseLLMHTTPHandler:
|
|||
async_httpx_client = client
|
||||
|
||||
# Get URL and params from provider config
|
||||
url, params = provider_config.transform_list_files_request(
|
||||
url, params = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.transform_list_files_request,
|
||||
purpose=purpose,
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -5298,7 +5322,9 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
yield page_files[: MAX_FILE_LIST_LIMIT - listed_count]
|
||||
listed_count += len(page_files)
|
||||
next_request = self._next_listing_request(latest_page, provider_config, litellm_params, listed_count)
|
||||
next_request = await run_off_loop_if_aws(
|
||||
provider_config, self._next_listing_request, latest_page, provider_config, litellm_params, listed_count
|
||||
)
|
||||
if next_request is None:
|
||||
return
|
||||
url, params = next_request
|
||||
|
|
@ -5524,7 +5550,9 @@ class BaseLLMHTTPHandler:
|
|||
headers: dict,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> "_PreparedFileContentRequest":
|
||||
url, params = provider_config.transform_file_content_request(
|
||||
url, params = await run_off_loop_if_aws(
|
||||
provider_config,
|
||||
provider_config.transform_file_content_request,
|
||||
file_content_request=file_content_request,
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
|
|
@ -10332,7 +10360,9 @@ class BaseLLMHTTPHandler:
|
|||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
|
||||
api_base: Final = vector_store_provider_config.get_complete_url(
|
||||
api_base: Final = await run_off_loop_if_aws(
|
||||
vector_store_provider_config,
|
||||
vector_store_provider_config.get_complete_url,
|
||||
api_base=litellm_params.api_base,
|
||||
litellm_params=dict(litellm_params),
|
||||
)
|
||||
|
|
@ -10366,7 +10396,7 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
all_optional_params: Final[dict[str, object]] = dict(litellm_params)
|
||||
all_optional_params.update(vector_store_search_optional_params or {})
|
||||
headers, signed_json_body = await sign_request_off_loop_if_aws(
|
||||
headers, signed_json_body = await run_off_loop_if_aws(
|
||||
vector_store_provider_config,
|
||||
vector_store_provider_config.sign_request,
|
||||
headers=headers,
|
||||
|
|
|
|||
|
|
@ -524,7 +524,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
|
|||
)
|
||||
if isinstance(request_view, VertexGeminiCountContextError):
|
||||
return count_context_error_response(request_view, request_model, model_to_use)
|
||||
count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
|
||||
count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools, model=model_to_use)
|
||||
if isinstance(count_context, VertexGeminiCountContextError):
|
||||
return count_context_error_response(count_context, request_model, model_to_use)
|
||||
blocks_error: Final = unconvertible_messages_error(
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ from litellm.llms.base_llm.google_genai.transformation import (
|
|||
from litellm.llms.vertex_ai.common_utils import (
|
||||
build_vertex_schema,
|
||||
supports_response_json_schema,
|
||||
with_integer_enums_that_fit,
|
||||
)
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig, VertexLLM
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -327,8 +328,11 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
|
|||
else:
|
||||
if json_schema_key is not None:
|
||||
generate_content_config_dict.pop(json_schema_key)
|
||||
generate_content_config_dict[schema_key] = build_vertex_schema(
|
||||
parameters=deepcopy(value), add_property_ordering=True
|
||||
generate_content_config_dict[schema_key] = with_integer_enums_that_fit(
|
||||
lambda send_integer_enums: build_vertex_schema(
|
||||
parameters=deepcopy(value), add_property_ordering=True, send_integer_enums=send_integer_enums
|
||||
),
|
||||
VertexGeminiConfig.is_gemini_3_or_newer(model),
|
||||
)
|
||||
|
||||
def transform_generate_content_request(
|
||||
|
|
|
|||
78
litellm/llms/github_copilot/github_host.py
Normal file
78
litellm/llms/github_copilot/github_host.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from litellm.constants import GITHUB_COPILOT_DEFAULT_HOST
|
||||
|
||||
from .authenticator import (
|
||||
DEFAULT_GITHUB_ACCESS_TOKEN_URL,
|
||||
DEFAULT_GITHUB_API_KEY_URL,
|
||||
DEFAULT_GITHUB_CLIENT_ID,
|
||||
DEFAULT_GITHUB_DEVICE_CODE_URL,
|
||||
)
|
||||
|
||||
_GHE_SUBDOMAIN_PATTERN: Final = re.compile(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GithubHostEndpoints:
|
||||
host: str
|
||||
device_code_url: str
|
||||
access_token_url: str
|
||||
api_key_url: str
|
||||
user_url: str
|
||||
client_id: str
|
||||
copilot_api_hosts: frozenset[str]
|
||||
copilot_api_host_suffix: str | None
|
||||
default_copilot_api_base: str
|
||||
|
||||
|
||||
def normalized_github_host(raw: object) -> str | None:
|
||||
if raw is None or raw == "":
|
||||
return GITHUB_COPILOT_DEFAULT_HOST
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
|
||||
host: Final = raw.lower()
|
||||
if host == GITHUB_COPILOT_DEFAULT_HOST:
|
||||
return host
|
||||
if not host.endswith(".ghe.com"):
|
||||
return None
|
||||
|
||||
subdomain: Final = host.removesuffix(".ghe.com")
|
||||
return host if _GHE_SUBDOMAIN_PATTERN.fullmatch(subdomain) is not None else None
|
||||
|
||||
|
||||
def github_host_endpoints(host: str) -> GithubHostEndpoints:
|
||||
normalized: Final = normalized_github_host(host)
|
||||
if normalized is None:
|
||||
raise ValueError("host must be github.com or a single-subdomain GHE.com hostname")
|
||||
client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID)
|
||||
if normalized == GITHUB_COPILOT_DEFAULT_HOST:
|
||||
return GithubHostEndpoints(
|
||||
host=normalized,
|
||||
device_code_url=os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL),
|
||||
access_token_url=os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL),
|
||||
api_key_url=os.getenv("GITHUB_COPILOT_API_KEY_URL", DEFAULT_GITHUB_API_KEY_URL),
|
||||
user_url="https://api.github.com/user",
|
||||
client_id=client_id,
|
||||
copilot_api_hosts=frozenset({"githubcopilot.com"}),
|
||||
copilot_api_host_suffix=".githubcopilot.com",
|
||||
default_copilot_api_base="https://api.githubcopilot.com",
|
||||
)
|
||||
|
||||
subdomain: Final = normalized.removesuffix(".ghe.com")
|
||||
api_host: Final = f"api.{subdomain}.ghe.com"
|
||||
copilot_api_host: Final = f"copilot-api.{subdomain}.ghe.com"
|
||||
return GithubHostEndpoints(
|
||||
host=normalized,
|
||||
device_code_url=f"https://{normalized}/login/device/code",
|
||||
access_token_url=f"https://{normalized}/login/oauth/access_token",
|
||||
api_key_url=f"https://{api_host}/copilot_internal/v2/token",
|
||||
user_url=f"https://{api_host}/user",
|
||||
client_id=client_id,
|
||||
copilot_api_hosts=frozenset({copilot_api_host}),
|
||||
copilot_api_host_suffix=None,
|
||||
default_copilot_api_base=f"https://{copilot_api_host}",
|
||||
)
|
||||
|
|
@ -5,8 +5,7 @@ touched on the per-user path.
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import os
|
||||
import hmac
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
|
|
@ -27,6 +26,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
|
|||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.constants import (
|
||||
GITHUB_COPILOT_AUTH_TYPE_KEY,
|
||||
GITHUB_COPILOT_HOST_KEY,
|
||||
GITHUB_COPILOT_PER_USER_AUTH_TYPE,
|
||||
GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS,
|
||||
)
|
||||
|
|
@ -38,14 +38,8 @@ from litellm.exceptions import (
|
|||
ServiceUnavailableError,
|
||||
)
|
||||
|
||||
from .authenticator import (
|
||||
DEFAULT_GITHUB_ACCESS_TOKEN_URL,
|
||||
DEFAULT_GITHUB_API_KEY_URL,
|
||||
DEFAULT_GITHUB_CLIENT_ID,
|
||||
DEFAULT_GITHUB_DEVICE_CODE_URL,
|
||||
github_api_headers,
|
||||
)
|
||||
from .common_utils import DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
from .authenticator import github_api_headers
|
||||
from .github_host import GithubHostEndpoints, github_host_endpoints, normalized_github_host
|
||||
|
||||
GITHUB_COPILOT_USER_SESSION_KWARG_KEY: Final = "github_copilot_user_session"
|
||||
_LLM_PROVIDER: Final = "github_copilot"
|
||||
|
|
@ -98,7 +92,7 @@ _SESSION_CACHE: Final = cast( # cast-ok: pins the untyped module cache to _Sess
|
|||
)
|
||||
_EXCHANGE_LOCKS: Final = tuple(threading.Lock() for _ in range(_SYNC_LOCK_COUNT))
|
||||
_IN_FLIGHT: Final[ # mutable-ok: single-flight dedup registry mutated under its own locking
|
||||
dict[str, "asyncio.Future[GithubCopilotUserSession]"]
|
||||
dict[str, tuple[str, "asyncio.Future[GithubCopilotUserSession]"]]
|
||||
] = {}
|
||||
|
||||
|
||||
|
|
@ -141,7 +135,7 @@ _DEVICE_CODE: Final = TypeAdapter(_DeviceCodePayload)
|
|||
_ACCESS_TOKEN_POLL: Final = TypeAdapter(_AccessTokenPollPayload)
|
||||
_GITHUB_USER: Final = TypeAdapter(_GithubUserPayload)
|
||||
_COPILOT_ENDPOINTS: Final = TypeAdapter(_CopilotEndpointsPayload)
|
||||
_CACHED_SESSION_ENTRY: Final = TypeAdapter(tuple[str, str, float])
|
||||
_CACHED_SESSION_ENTRY: Final = TypeAdapter(tuple[str, str, str, float])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True, repr=False)
|
||||
|
|
@ -178,53 +172,72 @@ class GithubCopilotDeviceFlowPoll:
|
|||
access_token: str | None = field(default=None, repr=False)
|
||||
|
||||
|
||||
def validated_copilot_api_base(endpoints_api: object) -> str:
|
||||
"""Only a genuine *.githubcopilot.com HTTPS endpoint may receive a Copilot
|
||||
token; anything else falls back to the default base."""
|
||||
def validated_copilot_api_base(endpoints_api: object, endpoints: GithubHostEndpoints) -> str:
|
||||
if not isinstance(endpoints_api, str):
|
||||
return DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
parsed: Final = urlparse(endpoints_api)
|
||||
return endpoints.default_copilot_api_base
|
||||
try:
|
||||
parsed: Final = urlparse(endpoints_api)
|
||||
port: Final = parsed.port
|
||||
except ValueError:
|
||||
return endpoints.default_copilot_api_base
|
||||
hostname: Final = (parsed.hostname or "").lower()
|
||||
if (
|
||||
parsed.scheme.lower() != "https"
|
||||
or not (hostname == "githubcopilot.com" or hostname.endswith(".githubcopilot.com"))
|
||||
or not (
|
||||
hostname in endpoints.copilot_api_hosts
|
||||
or (endpoints.copilot_api_host_suffix is not None and hostname.endswith(endpoints.copilot_api_host_suffix))
|
||||
)
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or port not in (None, 443)
|
||||
):
|
||||
return DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
try:
|
||||
port: Final = parsed.port
|
||||
except ValueError:
|
||||
return DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
if port not in (None, 443):
|
||||
return DEFAULT_GITHUB_COPILOT_API_BASE
|
||||
return endpoints.default_copilot_api_base
|
||||
return endpoints_api.rstrip("/")
|
||||
|
||||
|
||||
def _session_cache_key(user_id: str, github_token: str) -> str:
|
||||
return hashlib.sha256(f"{user_id}\x00{github_token}".encode()).hexdigest()
|
||||
def github_copilot_host_endpoints(credential_name: str) -> GithubHostEndpoints:
|
||||
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
|
||||
|
||||
credential: Final = CredentialAccessor.find_credential(credential_name)
|
||||
host_raw: Final[object] = (
|
||||
credential.credential_values.get(GITHUB_COPILOT_HOST_KEY) if credential is not None else None
|
||||
)
|
||||
host: Final = normalized_github_host(host_raw)
|
||||
if host is None:
|
||||
raise BadRequestError(
|
||||
message=f"Credential '{credential_name}' has an invalid {GITHUB_COPILOT_HOST_KEY}",
|
||||
llm_provider=_LLM_PROVIDER,
|
||||
model="",
|
||||
)
|
||||
return github_host_endpoints(host)
|
||||
|
||||
|
||||
def evict_copilot_user_session(user_id: str, github_token: str) -> None:
|
||||
_SESSION_CACHE.delete_cache(_session_cache_key(user_id, github_token))
|
||||
def _session_cache_key(user_id: str, credential_name: str, github_host: str) -> str:
|
||||
return f"{user_id}\x00{credential_name}\x00{github_host}"
|
||||
|
||||
|
||||
def _cached_session(cache_key: str) -> GithubCopilotUserSession | None:
|
||||
def evict_copilot_user_session(user_id: str, credential_name: str, github_host: str) -> None:
|
||||
_SESSION_CACHE.delete_cache(_session_cache_key(user_id, credential_name, github_host))
|
||||
|
||||
|
||||
def _cached_session(cache_key: str, github_token: str) -> GithubCopilotUserSession | None:
|
||||
cached: Final[object] = _SESSION_CACHE.get_cache(cache_key)
|
||||
try:
|
||||
token, api_base, expires_at = _CACHED_SESSION_ENTRY.validate_python(cached)
|
||||
cached_github_token, token, api_base, expires_at = _CACHED_SESSION_ENTRY.validate_python(cached)
|
||||
except ValidationError:
|
||||
return None
|
||||
if not hmac.compare_digest(cached_github_token, github_token):
|
||||
return None
|
||||
if expires_at - GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS <= time.time():
|
||||
return None
|
||||
return GithubCopilotUserSession(token=token, api_base=api_base)
|
||||
|
||||
|
||||
def _cache_session(cache_key: str, session: GithubCopilotUserSession, expires_at: float) -> None:
|
||||
def _cache_session(cache_key: str, github_token: str, session: GithubCopilotUserSession, expires_at: float) -> None:
|
||||
ttl: Final = expires_at - GITHUB_COPILOT_USER_TOKEN_SAFETY_MARGIN_SECONDS - time.time()
|
||||
if ttl <= 0:
|
||||
return
|
||||
_SESSION_CACHE.set_cache(cache_key, (session.token, session.api_base, expires_at), ttl=ttl)
|
||||
_SESSION_CACHE.set_cache(cache_key, (github_token, session.token, session.api_base, expires_at), ttl=ttl)
|
||||
|
||||
|
||||
def _reject_connection(credential_name: str) -> CallerCredentialAuthenticationError:
|
||||
|
|
@ -238,7 +251,13 @@ def _reject_connection(credential_name: str) -> CallerCredentialAuthenticationEr
|
|||
)
|
||||
|
||||
|
||||
def _session_from_response(response: httpx.Response, credential_name: str, cache_key: str) -> GithubCopilotUserSession:
|
||||
def _session_from_response(
|
||||
response: httpx.Response,
|
||||
github_token: str,
|
||||
credential_name: str,
|
||||
cache_key: str,
|
||||
endpoints: GithubHostEndpoints,
|
||||
) -> GithubCopilotUserSession:
|
||||
if response.status_code in (401, 403, 404):
|
||||
_SESSION_CACHE.delete_cache(cache_key)
|
||||
raise _reject_connection(credential_name)
|
||||
|
|
@ -263,15 +282,17 @@ def _session_from_response(response: httpx.Response, credential_name: str, cache
|
|||
model="",
|
||||
) from e
|
||||
|
||||
endpoints: Final[object] = payload.get("endpoints")
|
||||
payload_endpoints: Final[object] = payload.get("endpoints")
|
||||
endpoints_api: Final[object] = (
|
||||
_COPILOT_ENDPOINTS.validate_python(endpoints).get("api") if isinstance(endpoints, Mapping) else None
|
||||
_COPILOT_ENDPOINTS.validate_python(payload_endpoints).get("api")
|
||||
if isinstance(payload_endpoints, Mapping)
|
||||
else None
|
||||
)
|
||||
api_base: Final = validated_copilot_api_base(endpoints_api)
|
||||
api_base: Final = validated_copilot_api_base(endpoints_api, endpoints)
|
||||
session: Final = GithubCopilotUserSession(token=payload["token"], api_base=api_base)
|
||||
expires_at: Final = _expires_at_seconds(payload.get("expires_at"))
|
||||
if expires_at > 0:
|
||||
_cache_session(cache_key, session, expires_at)
|
||||
_cache_session(cache_key, github_token, session, expires_at)
|
||||
return session
|
||||
|
||||
|
||||
|
|
@ -303,7 +324,12 @@ def _expires_at_seconds(raw: object) -> float:
|
|||
return 0.0
|
||||
|
||||
|
||||
def _do_exchange(github_token: str, credential_name: str, cache_key: str) -> GithubCopilotUserSession:
|
||||
def _do_exchange(
|
||||
github_token: str,
|
||||
credential_name: str,
|
||||
cache_key: str,
|
||||
endpoints: GithubHostEndpoints,
|
||||
) -> GithubCopilotUserSession:
|
||||
import litellm
|
||||
|
||||
try:
|
||||
|
|
@ -311,7 +337,7 @@ def _do_exchange(github_token: str, credential_name: str, cache_key: str) -> Git
|
|||
_SyncGetClient, litellm.module_level_client
|
||||
)
|
||||
response: Final = client.get(
|
||||
DEFAULT_GITHUB_API_KEY_URL,
|
||||
endpoints.api_key_url,
|
||||
headers=github_api_headers(github_token),
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
@ -320,7 +346,7 @@ def _do_exchange(github_token: str, credential_name: str, cache_key: str) -> Git
|
|||
llm_provider=_LLM_PROVIDER,
|
||||
model="",
|
||||
) from e
|
||||
return _session_from_response(response, credential_name, cache_key)
|
||||
return _session_from_response(response, github_token, credential_name, cache_key, endpoints)
|
||||
|
||||
|
||||
def _new_async_github_client() -> _AsyncGitHubClient:
|
||||
|
|
@ -332,11 +358,16 @@ def _new_async_github_client() -> _AsyncGitHubClient:
|
|||
)(_LLM_PROVIDER)
|
||||
|
||||
|
||||
async def _do_aexchange(github_token: str, credential_name: str, cache_key: str) -> GithubCopilotUserSession:
|
||||
async def _do_aexchange(
|
||||
github_token: str,
|
||||
credential_name: str,
|
||||
cache_key: str,
|
||||
endpoints: GithubHostEndpoints,
|
||||
) -> GithubCopilotUserSession:
|
||||
client: Final = _new_async_github_client()
|
||||
try:
|
||||
response: Final = await client.get(
|
||||
DEFAULT_GITHUB_API_KEY_URL,
|
||||
endpoints.api_key_url,
|
||||
headers=github_api_headers(github_token),
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
@ -345,43 +376,50 @@ async def _do_aexchange(github_token: str, credential_name: str, cache_key: str)
|
|||
llm_provider=_LLM_PROVIDER,
|
||||
model="",
|
||||
) from e
|
||||
return _session_from_response(response, credential_name, cache_key)
|
||||
return _session_from_response(response, github_token, credential_name, cache_key, endpoints)
|
||||
|
||||
|
||||
def exchange_github_token(
|
||||
user_id: str,
|
||||
github_token: str,
|
||||
credential_name: str,
|
||||
endpoints: GithubHostEndpoints,
|
||||
) -> GithubCopilotUserSession:
|
||||
cache_key: Final = _session_cache_key(user_id, github_token)
|
||||
cached: Final = _cached_session(cache_key)
|
||||
cache_key: Final = _session_cache_key(user_id, credential_name, endpoints.host)
|
||||
cached: Final = _cached_session(cache_key, github_token)
|
||||
if cached is not None:
|
||||
return cached
|
||||
lock: Final = _EXCHANGE_LOCKS[int(cache_key, 16) % _SYNC_LOCK_COUNT]
|
||||
lock: Final = _EXCHANGE_LOCKS[hash(cache_key) % _SYNC_LOCK_COUNT]
|
||||
with lock:
|
||||
rechecked: Final = _cached_session(cache_key)
|
||||
rechecked: Final = _cached_session(cache_key, github_token)
|
||||
if rechecked is not None:
|
||||
return rechecked
|
||||
return _do_exchange(github_token, credential_name, cache_key)
|
||||
return _do_exchange(github_token, credential_name, cache_key, endpoints)
|
||||
|
||||
|
||||
async def aexchange_github_token(
|
||||
user_id: str,
|
||||
github_token: str,
|
||||
credential_name: str,
|
||||
endpoints: GithubHostEndpoints,
|
||||
) -> GithubCopilotUserSession:
|
||||
cache_key: Final = _session_cache_key(user_id, github_token)
|
||||
cached: Final = _cached_session(cache_key)
|
||||
cache_key: Final = _session_cache_key(user_id, credential_name, endpoints.host)
|
||||
cached: Final = _cached_session(cache_key, github_token)
|
||||
if cached is not None:
|
||||
return cached
|
||||
loop: Final = asyncio.get_running_loop()
|
||||
in_flight: Final = _IN_FLIGHT.get(cache_key)
|
||||
if in_flight is not None and not in_flight.done() and in_flight.get_loop() is loop:
|
||||
return await asyncio.shield(in_flight)
|
||||
in_flight_entry: Final = _IN_FLIGHT.get(cache_key)
|
||||
if in_flight_entry is not None:
|
||||
in_flight_token, in_flight = in_flight_entry
|
||||
if not hmac.compare_digest(in_flight_token, github_token):
|
||||
return await _do_aexchange(github_token, credential_name, cache_key, endpoints)
|
||||
if not in_flight.done() and in_flight.get_loop() is loop:
|
||||
return await asyncio.shield(in_flight)
|
||||
future: Final = loop.create_future()
|
||||
_IN_FLIGHT[cache_key] = future
|
||||
entry: Final = (github_token, future)
|
||||
_IN_FLIGHT[cache_key] = entry
|
||||
try:
|
||||
session: Final = await _do_aexchange(github_token, credential_name, cache_key)
|
||||
session: Final = await _do_aexchange(github_token, credential_name, cache_key, endpoints)
|
||||
except BaseException as e:
|
||||
if not future.done():
|
||||
future.set_exception(e)
|
||||
|
|
@ -392,19 +430,17 @@ async def aexchange_github_token(
|
|||
future.set_result(session)
|
||||
return session
|
||||
finally:
|
||||
if _IN_FLIGHT.get(cache_key) is future:
|
||||
if _IN_FLIGHT.get(cache_key) is entry:
|
||||
_IN_FLIGHT.pop(cache_key, None)
|
||||
|
||||
|
||||
async def astart_device_flow() -> GithubCopilotDeviceFlowStart:
|
||||
async def astart_device_flow(endpoints: GithubHostEndpoints) -> GithubCopilotDeviceFlowStart:
|
||||
client: Final = _new_async_github_client()
|
||||
device_code_url: Final = os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL)
|
||||
client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID)
|
||||
try:
|
||||
response: Final = await client.post(
|
||||
device_code_url,
|
||||
endpoints.device_code_url,
|
||||
headers=github_api_headers(),
|
||||
json={"client_id": client_id, "scope": _DEVICE_FLOW_SCOPE},
|
||||
json={"client_id": endpoints.client_id, "scope": _DEVICE_FLOW_SCOPE},
|
||||
)
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise ServiceUnavailableError(
|
||||
|
|
@ -435,16 +471,14 @@ async def astart_device_flow() -> GithubCopilotDeviceFlowStart:
|
|||
)
|
||||
|
||||
|
||||
async def apoll_device_flow(device_code: str) -> GithubCopilotDeviceFlowPoll:
|
||||
async def apoll_device_flow(device_code: str, endpoints: GithubHostEndpoints) -> GithubCopilotDeviceFlowPoll:
|
||||
client: Final = _new_async_github_client()
|
||||
access_token_url: Final = os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL)
|
||||
client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID)
|
||||
try:
|
||||
response: Final = await client.post(
|
||||
access_token_url,
|
||||
endpoints.access_token_url,
|
||||
headers=github_api_headers(),
|
||||
json={
|
||||
"client_id": client_id,
|
||||
"client_id": endpoints.client_id,
|
||||
"device_code": device_code,
|
||||
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
|
||||
},
|
||||
|
|
@ -500,11 +534,11 @@ def _poll_payload_from_response(response: httpx.Response) -> GithubCopilotDevice
|
|||
)
|
||||
|
||||
|
||||
async def afetch_github_login(github_token: str) -> str:
|
||||
async def afetch_github_login(github_token: str, endpoints: GithubHostEndpoints) -> str:
|
||||
client: Final = _new_async_github_client()
|
||||
try:
|
||||
response: Final = await client.get(
|
||||
"https://api.github.com/user",
|
||||
endpoints.user_url,
|
||||
headers=github_api_headers(github_token),
|
||||
)
|
||||
except Exception as e:
|
||||
|
|
@ -530,11 +564,17 @@ async def afetch_github_login(github_token: str) -> str:
|
|||
return payload["login"]
|
||||
|
||||
|
||||
async def acheck_copilot_seat(user_id: str, github_token: str, credential_name: str) -> GithubCopilotUserSession:
|
||||
async def acheck_copilot_seat(
|
||||
user_id: str,
|
||||
github_token: str,
|
||||
credential_name: str,
|
||||
endpoints: GithubHostEndpoints,
|
||||
) -> GithubCopilotUserSession:
|
||||
return await aexchange_github_token(
|
||||
user_id=user_id,
|
||||
github_token=github_token,
|
||||
credential_name=credential_name,
|
||||
endpoints=endpoints,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -567,7 +607,11 @@ def _connect_error(credential_name: str) -> CallerCredentialAuthenticationError:
|
|||
)
|
||||
|
||||
|
||||
def _caller_connection(kwargs: Mapping[str, object], credential_name: str) -> tuple[str, str]:
|
||||
def _caller_connection(
|
||||
kwargs: Mapping[str, object],
|
||||
credential_name: str,
|
||||
endpoints: GithubHostEndpoints,
|
||||
) -> tuple[str, str]:
|
||||
secret_fields_raw: Final[object] = kwargs.get("secret_fields")
|
||||
secret_fields: Final = (
|
||||
cast(Mapping[object, object], secret_fields_raw) # cast-ok: value is Mapping-checked or dict-shaped at runtime
|
||||
|
|
@ -585,8 +629,21 @@ def _caller_connection(kwargs: Mapping[str, object], credential_name: str) -> tu
|
|||
if isinstance(credentials_raw, Mapping)
|
||||
else None
|
||||
)
|
||||
token: Final[object] = credentials.get(credential_name) if credentials is not None else None
|
||||
if not isinstance(user_id, str) or not user_id or not isinstance(token, str) or not token:
|
||||
connection_raw: Final[object] = credentials.get(credential_name) if credentials is not None else None
|
||||
connection: Final = (
|
||||
cast(Mapping[object, object], connection_raw) # cast-ok: runtime Mapping check validates the stored connection
|
||||
if isinstance(connection_raw, Mapping)
|
||||
else None
|
||||
)
|
||||
token: Final[object] = connection.get("access_token") if connection is not None else None
|
||||
github_host: Final = normalized_github_host(connection.get("github_host") if connection is not None else None)
|
||||
if (
|
||||
not isinstance(user_id, str)
|
||||
or not user_id
|
||||
or not isinstance(token, str)
|
||||
or not token
|
||||
or github_host != endpoints.host
|
||||
):
|
||||
raise _connect_error(credential_name)
|
||||
return user_id, token
|
||||
|
||||
|
|
@ -705,11 +762,13 @@ def attach_github_copilot_user_session(
|
|||
if isinstance(kwargs.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY), GithubCopilotUserSession):
|
||||
return
|
||||
credential_name: Final = credential_name_obj if isinstance(credential_name_obj, str) else ""
|
||||
user_id, github_token = _caller_connection(kwargs, credential_name)
|
||||
endpoints: Final = github_copilot_host_endpoints(credential_name)
|
||||
user_id, github_token = _caller_connection(kwargs, credential_name, endpoints)
|
||||
session: Final = exchange_github_token(
|
||||
user_id=user_id,
|
||||
github_token=github_token,
|
||||
credential_name=credential_name,
|
||||
endpoints=endpoints,
|
||||
)
|
||||
kwargs[GITHUB_COPILOT_USER_SESSION_KWARG_KEY] = session # rebind-ok: kwargs is the request mutation channel
|
||||
_strip_caller_authorization(kwargs)
|
||||
|
|
@ -725,11 +784,13 @@ async def aattach_github_copilot_user_session(
|
|||
if isinstance(kwargs.get(GITHUB_COPILOT_USER_SESSION_KWARG_KEY), GithubCopilotUserSession):
|
||||
return
|
||||
credential_name: Final = credential_name_obj if isinstance(credential_name_obj, str) else ""
|
||||
user_id, github_token = _caller_connection(kwargs, credential_name)
|
||||
endpoints: Final = github_copilot_host_endpoints(credential_name)
|
||||
user_id, github_token = _caller_connection(kwargs, credential_name, endpoints)
|
||||
session: Final = await aexchange_github_token(
|
||||
user_id=user_id,
|
||||
github_token=github_token,
|
||||
credential_name=credential_name,
|
||||
endpoints=endpoints,
|
||||
)
|
||||
kwargs[GITHUB_COPILOT_USER_SESSION_KWARG_KEY] = session # rebind-ok: kwargs is the request mutation channel
|
||||
_strip_caller_authorization(kwargs)
|
||||
|
|
|
|||
|
|
@ -8,17 +8,31 @@ repo's own opencode.json cannot redirect model calls. The token is never in argv
|
|||
the config references it with `{file:<private_dir>/token}`. XDG dirs point at a persisted
|
||||
LiteLLM-owned root so the user's opencode config and auth are never read, and the session
|
||||
DB outlives a session for resume. Verified against opencode 1.14.41.
|
||||
|
||||
opencode files each session under the folder it ran in and only streams a resumed session's
|
||||
events from that folder, so resuming one filed elsewhere (by an older LiteLLM that left PWD
|
||||
unset, or from another sandbox) hangs. The first turn after a resume runs `opencode export` in
|
||||
the sandbox, and when the session is filed elsewhere it imports a copy under fresh ids, filed
|
||||
under the sandbox folder, and continues the copy. Import overwrites the folder of a session id it
|
||||
already has, so reusing the ids would break the original wherever it was saved. Before 1.15.6,
|
||||
import files a session under the folder named in its JSON, so the copy names the sandbox folder.
|
||||
A failed or timed-out export or import fails that turn instead of hanging, and the next turn
|
||||
tries again. Verified against opencode 1.14.41, 1.15.5, 1.15.6 and 1.19.3.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
import secrets
|
||||
import string
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.harness.errors import CapabilityUnsupported, HarnessError, OptionsMismatch
|
||||
from litellm.harness.options import OpenCodeOptions
|
||||
from litellm.harness.types import (
|
||||
|
|
@ -33,6 +47,7 @@ from litellm.harness.types import (
|
|||
)
|
||||
from litellm.llms.base_llm.harness.transformation import (
|
||||
BaseCLIHarnessConfig,
|
||||
HarnessSessionCopy,
|
||||
HarnessSessionSetup,
|
||||
HarnessTurnError,
|
||||
HarnessTurnRequest,
|
||||
|
|
@ -57,6 +72,10 @@ OPENCODE_PROVIDER_NPM: Final = "@ai-sdk/openai-compatible"
|
|||
# A fixed title skips opencode's extra title-generation model call on the first turn.
|
||||
OPENCODE_SESSION_TITLE: Final = "litellm-harness"
|
||||
TOKEN_FILENAME: Final = "token"
|
||||
RESUME_COPY_FILENAME: Final = "resume-session.json"
|
||||
ID_ORDERED_PREFIX_LENGTH: Final = 16
|
||||
ID_RANDOM_LENGTH: Final = 14
|
||||
ID_RANDOM_ALPHABET: Final = string.ascii_letters + string.digits
|
||||
INSTRUCTIONS_FILENAME: Final = "instructions.md"
|
||||
XDG_DIRNAME: Final = "xdg"
|
||||
XDG_SUBDIRS: Final = ("config", "data", "state", "cache")
|
||||
|
|
@ -162,6 +181,94 @@ class OpenCodeStreamState:
|
|||
step_texts: Sequence[str] = ()
|
||||
|
||||
|
||||
class ExportedPart(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
class ExportedMessageInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str
|
||||
|
||||
|
||||
class ExportedMessage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
info: ExportedMessageInfo
|
||||
parts: tuple[ExportedPart, ...]
|
||||
|
||||
|
||||
class ExportedSessionInfo(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str
|
||||
directory: str
|
||||
|
||||
|
||||
class ExportedSession(BaseModel):
|
||||
"""The ids and folder of an `opencode export`; every other field is copied as is."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
info: ExportedSessionInfo
|
||||
messages: tuple[ExportedMessage, ...]
|
||||
|
||||
def ids(self) -> Iterator[str]:
|
||||
yield self.info.id
|
||||
for message in self.messages:
|
||||
yield message.info.id
|
||||
yield from (part.id for part in message.parts)
|
||||
|
||||
|
||||
EXPORT_JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
||||
def fresh_id(old: str) -> str:
|
||||
"""Keeps the type prefix and time-ordered part of an opencode id, so the copy sorts like the original."""
|
||||
suffix: Final = "".join(secrets.choice(ID_RANDOM_ALPHABET) for _ in range(ID_RANDOM_LENGTH))
|
||||
return f"{old[:ID_ORDERED_PREFIX_LENGTH]}{suffix}"
|
||||
|
||||
|
||||
def with_id_fields_replaced(value: JsonValue, fresh: Mapping[str, str], key: str = "") -> JsonValue:
|
||||
match value:
|
||||
case str() if key == "id" or key.endswith("ID"):
|
||||
return fresh.get(value, value)
|
||||
case list():
|
||||
return [with_id_fields_replaced(item, fresh, key) for item in value]
|
||||
case dict():
|
||||
return {name: with_id_fields_replaced(item, fresh, name) for name, item in value.items()}
|
||||
case _:
|
||||
return value
|
||||
|
||||
|
||||
def filed_under(exported: JsonValue, workdir: str) -> JsonValue:
|
||||
match exported:
|
||||
case {"info": dict() as info}:
|
||||
return {**exported, "info": {**info, "directory": workdir}}
|
||||
case _:
|
||||
return exported
|
||||
|
||||
|
||||
def session_copy(exported: str, path: str, workdir: str) -> HarnessSessionCopy | None:
|
||||
"""A copy of an exported session filed under workdir, or None when the session is already filed there."""
|
||||
try:
|
||||
raw: Final = EXPORT_JSON.validate_json(exported)
|
||||
session: Final = ExportedSession.model_validate(raw)
|
||||
except ValidationError as exc:
|
||||
raise HarnessError(f"opencode export did not print a session: {exc.errors()[0]['msg']}") from exc
|
||||
if session.info.directory == workdir:
|
||||
return None
|
||||
fresh: Final = MappingProxyType({old: fresh_id(old) for old in session.ids()})
|
||||
return HarnessSessionCopy(
|
||||
session_id=fresh[session.info.id],
|
||||
path=path,
|
||||
data=json.dumps(filed_under(with_id_fields_replaced(raw, fresh), workdir)).encode("utf-8"),
|
||||
import_command=(OPENCODE_BINARY, "--pure", "import", path),
|
||||
)
|
||||
|
||||
|
||||
def _as_dict(value: object) -> Mapping[str, Any]:
|
||||
return value if isinstance(value, dict) else MappingProxyType({})
|
||||
|
||||
|
|
@ -362,6 +469,12 @@ class OpenCodeHarnessConfig(BaseCLIHarnessConfig):
|
|||
# The prompt goes on stdin; opencode appends non-TTY stdin to the message.
|
||||
return HarnessTurnRequest(argv=argv, env=setup.env, stdin=turn_prompt(ctx, prompt), cwd=ctx.sandbox.workdir)
|
||||
|
||||
def transform_resume_export_command(self, native_session_id: str) -> Sequence[str]:
|
||||
return (OPENCODE_BINARY, "--pure", "export", native_session_id)
|
||||
|
||||
def transform_resume_copy(self, exported: str, private_dir: str, workdir: str) -> HarnessSessionCopy | None:
|
||||
return session_copy(exported, f"{private_dir}/{RESUME_COPY_FILENAME}", workdir)
|
||||
|
||||
def create_stream_state(self) -> OpenCodeStreamState:
|
||||
return OpenCodeStreamState()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from itertools import accumulate, chain, repeat, takewhile
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final, Literal, cast, get_type_hints
|
||||
from typing import Any, Final, Literal, TypeVar, cast, get_type_hints
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
|
@ -666,12 +667,89 @@ def _fix_enum_empty_strings(schema, depth=0):
|
|||
_fix_enum_empty_strings(items, depth=depth + 1)
|
||||
|
||||
|
||||
def _fix_enum_types(schema, depth=0):
|
||||
"""Remove `enum` fields when the schema type is not string.
|
||||
GEMINI_MAX_ENUM_CHARACTERS: Final = 2000
|
||||
_GEMINI_EXACT_INT_MAX: Final = 2**53 - 1
|
||||
|
||||
Gemini / Vertex APIs only allow enums for string-typed fields. When an enum
|
||||
is present on a non-string typed property (or when `anyOf` types do not
|
||||
include a string type), remove the enum to avoid provider validation errors.
|
||||
|
||||
def _gemini_integer_enum_value(value: object) -> str | None:
|
||||
match value:
|
||||
case bool():
|
||||
return None
|
||||
case int() if -_GEMINI_EXACT_INT_MAX <= value <= _GEMINI_EXACT_INT_MAX:
|
||||
return str(int(value))
|
||||
case str() if (
|
||||
re.fullmatch(r"-?[0-9]{1,16}", value) and -_GEMINI_EXACT_INT_MAX <= int(value) <= _GEMINI_EXACT_INT_MAX
|
||||
):
|
||||
return str(int(value))
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _gemini_integer_enum(values: Sequence[object]) -> tuple[str, ...] | None:
|
||||
if not values:
|
||||
return None
|
||||
converted: Final = tuple(text for value in values if (text := _gemini_integer_enum_value(value)) is not None)
|
||||
return converted if len(converted) == len(values) else None
|
||||
|
||||
|
||||
def _json_children(node: JsonValue) -> Iterable[JsonValue]:
|
||||
match node:
|
||||
case dict():
|
||||
return node.values()
|
||||
case list():
|
||||
return node
|
||||
case _:
|
||||
return ()
|
||||
|
||||
|
||||
def _next_json_level(level: tuple[JsonValue, ...], _: None) -> tuple[JsonValue, ...]:
|
||||
return tuple(chain.from_iterable(_json_children(node) for node in level))
|
||||
|
||||
|
||||
def _enum_size(node: JsonValue) -> tuple[bool, int] | None:
|
||||
match node:
|
||||
case {"enum": list() as values}:
|
||||
return (str(node.get("type")).lower() == "integer", sum(len(str(value)) for value in values))
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _integer_enums_fit(schema: JsonValue) -> bool:
|
||||
levels: Final = takewhile(bool, accumulate(repeat(None), _next_json_level, initial=(schema,)))
|
||||
enum_sizes: Final = tuple(size for node in chain.from_iterable(levels) if (size := _enum_size(node)) is not None)
|
||||
return (
|
||||
not any(is_integer for is_integer, _ in enum_sizes)
|
||||
or sum(characters for _, characters in enum_sizes) <= GEMINI_MAX_ENUM_CHARACTERS
|
||||
)
|
||||
|
||||
|
||||
_GeminiRequestPart: Final = TypeVar("_GeminiRequestPart")
|
||||
|
||||
|
||||
def with_integer_enums_that_fit(
|
||||
build: Callable[[bool], _GeminiRequestPart], send_integer_enums: bool
|
||||
) -> _GeminiRequestPart:
|
||||
if not send_integer_enums:
|
||||
return build(False)
|
||||
with_integer_enums: Final = build(True)
|
||||
if _integer_enums_fit(
|
||||
cast(JsonValue, with_integer_enums) # cast-ok: built tools and schemas are JSON request data
|
||||
):
|
||||
return with_integer_enums
|
||||
verbose_logger.debug(
|
||||
"Gemini integer enums dropped: the request's enum values exceed %s characters", GEMINI_MAX_ENUM_CHARACTERS
|
||||
)
|
||||
return build(False)
|
||||
|
||||
|
||||
def _fix_enum_types(schema, depth=0, send_integer_enums: bool = False):
|
||||
"""Make `enum` fields acceptable to Gemini / Vertex.
|
||||
|
||||
Gemini / Vertex APIs only accept string enum values. With `send_integer_enums`,
|
||||
an integer enum of values a double holds exactly is sent as type INTEGER with the
|
||||
values as strings; the model still returns an integer.
|
||||
On any other non-string typed property (or when `anyOf` types do not include a
|
||||
string type), the enum is removed to avoid provider validation errors.
|
||||
"""
|
||||
if depth > DEFAULT_MAX_RECURSE_DEPTH:
|
||||
raise ValueError(f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema.")
|
||||
|
|
@ -696,26 +774,37 @@ def _fix_enum_types(schema, depth=0):
|
|||
break
|
||||
|
||||
if not keep_enum:
|
||||
schema.pop("enum", None)
|
||||
is_integer_type: Final = isinstance(schema_type, str) and schema_type.lower() == "integer"
|
||||
integer_enum: Final = (
|
||||
_gemini_integer_enum(cast(Sequence[object], schema["enum"])) # cast-ok: an untyped JSON list
|
||||
if send_integer_enums and is_integer_type
|
||||
else None
|
||||
)
|
||||
if integer_enum is not None:
|
||||
schema["enum"] = list(integer_enum) # rebind-ok: this function edits the schema in place by design
|
||||
else:
|
||||
schema.pop("enum", None)
|
||||
|
||||
# Recurse into nested structures
|
||||
properties: Final = schema.get("properties", None)
|
||||
if properties is not None:
|
||||
for _, value in properties.items():
|
||||
_fix_enum_types(value, depth=depth + 1)
|
||||
_fix_enum_types(value, depth=depth + 1, send_integer_enums=send_integer_enums)
|
||||
|
||||
items: Final = schema.get("items", None)
|
||||
if items is not None:
|
||||
_fix_enum_types(items, depth=depth + 1)
|
||||
_fix_enum_types(items, depth=depth + 1, send_integer_enums=send_integer_enums)
|
||||
|
||||
anyof = schema.get("anyOf", None)
|
||||
if anyof is not None and isinstance(anyof, list):
|
||||
for item in anyof:
|
||||
if isinstance(item, dict):
|
||||
_fix_enum_types(item, depth=depth + 1)
|
||||
_fix_enum_types(item, depth=depth + 1, send_integer_enums=send_integer_enums)
|
||||
|
||||
|
||||
def build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
|
||||
def build_vertex_schema(
|
||||
parameters: dict, add_property_ordering: bool = False, send_integer_enums: bool = False
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
This is a modified version of https://github.com/google-gemini/generative-ai-python/blob/8f77cc6ac99937cd3a81299ecf79608b91b06bbb/google/generativeai/types/content_types.py#L419
|
||||
|
||||
|
|
@ -725,6 +814,7 @@ def build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
|
|||
parameters: dict - the json schema to build from
|
||||
add_property_ordering: bool - whether to add propertyOrdering to the schema. This is only applicable to schemas for structured outputs. See
|
||||
set_schema_property_ordering for more details.
|
||||
send_integer_enums: bool - whether to send integer enums with string values instead of removing them
|
||||
Returns:
|
||||
parameters: dict - the input parameters, modified in place
|
||||
"""
|
||||
|
|
@ -750,7 +840,7 @@ def build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
|
|||
_fix_enum_empty_strings(parameters)
|
||||
|
||||
# Remove enums for non-string typed fields (Gemini requires enum only on strings)
|
||||
_fix_enum_types(parameters)
|
||||
_fix_enum_types(parameters, send_integer_enums=send_integer_enums)
|
||||
|
||||
# Handle empty items objects
|
||||
process_items(parameters)
|
||||
|
|
@ -1068,6 +1158,17 @@ def _convert_schema_types(schema, depth=0):
|
|||
if field in schema:
|
||||
item_schema[field] = deepcopy(schema[field])
|
||||
any_of.append(item_schema)
|
||||
elif t == "integer" and "enum" in schema and isinstance(schema["enum"], list):
|
||||
any_of.append(
|
||||
{
|
||||
"type": t,
|
||||
"enum": [
|
||||
value
|
||||
for value in cast(list[object], schema["enum"]) # cast-ok: an untyped JSON list
|
||||
if value is not None
|
||||
],
|
||||
}
|
||||
)
|
||||
else:
|
||||
# For primitive types, only include the type
|
||||
any_of.append({"type": t})
|
||||
|
|
@ -1454,7 +1555,9 @@ class VertexAITokenCounter(BaseTokenCounter):
|
|||
)
|
||||
if isinstance(request_view, VertexGeminiCountContextError):
|
||||
return count_context_error_response(request_view, request_model, model_to_use)
|
||||
count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
|
||||
count_context: Final = vertex_gemini_count_context(
|
||||
system=request_view.system, tools=tools, model=model_to_use
|
||||
)
|
||||
if isinstance(count_context, VertexGeminiCountContextError):
|
||||
return count_context_error_response(count_context, request_model, model_to_use)
|
||||
blocks_error: Final = unconvertible_messages_error(
|
||||
|
|
|
|||
|
|
@ -78,6 +78,7 @@ def _is_function_tool(tool: Mapping[str, JsonValue]) -> bool:
|
|||
|
||||
def _map_function_tools(
|
||||
tools: object | None,
|
||||
model: str,
|
||||
) -> tuple[Mapping[str, object], ...] | None | VertexGeminiCountContextError:
|
||||
try:
|
||||
tool_values: Final = _TOOLS.validate_python(tools or ())
|
||||
|
|
@ -89,15 +90,21 @@ def _map_function_tools(
|
|||
if not all(_has_object_schema(tool) for tool in function_tools):
|
||||
return _INVALID_TOOLS
|
||||
|
||||
from litellm.llms.vertex_ai.common_utils import with_integer_enums_that_fit
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
|
||||
|
||||
chat_tools: Final = tuple(
|
||||
chat_tool for tool in function_tools for chat_tool in chat_completion_tools_for(tool)
|
||||
) # comprehension-ok: flattens each tool's translation
|
||||
config: Final = VertexGeminiConfig()
|
||||
try:
|
||||
mapped_tools: Final = VertexGeminiConfig().map_function(
|
||||
value=[dict(tool) for tool in chat_tools],
|
||||
optional_params={},
|
||||
mapped_tools: Final = with_integer_enums_that_fit(
|
||||
lambda send_integer_enums: config.map_function(
|
||||
value=[dict(tool) for tool in chat_tools],
|
||||
optional_params={},
|
||||
send_integer_enums=send_integer_enums,
|
||||
),
|
||||
VertexGeminiConfig.is_gemini_3_or_newer(model),
|
||||
)
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return _INVALID_TOOLS
|
||||
|
|
@ -110,11 +117,12 @@ def _map_function_tools(
|
|||
def vertex_gemini_count_context(
|
||||
system: object | None,
|
||||
tools: object | None,
|
||||
model: str,
|
||||
) -> VertexGeminiCountContext | VertexGeminiCountContextError:
|
||||
system_parts: Final = _system_parts(system)
|
||||
if isinstance(system_parts, VertexGeminiCountContextError):
|
||||
return system_parts
|
||||
function_tools: Final = _map_function_tools(tools)
|
||||
function_tools: Final = _map_function_tools(tools, model)
|
||||
if isinstance(function_tools, VertexGeminiCountContextError):
|
||||
return function_tools
|
||||
return VertexGeminiCountContext(
|
||||
|
|
|
|||
|
|
@ -1118,6 +1118,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
|
|||
non_default_params=openai_request_body,
|
||||
optional_params={},
|
||||
drop_params=False,
|
||||
allow_integer_enums=False,
|
||||
)
|
||||
return vertex_params
|
||||
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ from ..common_utils import (
|
|||
build_json_schema,
|
||||
build_vertex_schema,
|
||||
supports_response_json_schema,
|
||||
with_integer_enums_that_fit,
|
||||
)
|
||||
from ..context_caching.known_cached_contents import known_cached_contents
|
||||
from ..context_caching.rejected_cache_retry import (
|
||||
|
|
@ -692,13 +693,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
|
||||
return googleSearch, googleSearchRetrieval, enterpriseWebSearch, urlContext
|
||||
|
||||
def _map_function(self, value: list[dict], optional_params: dict) -> list[Tools]:
|
||||
def _map_function(self, value: list[dict], optional_params: dict, send_integer_enums: bool = False) -> list[Tools]:
|
||||
"""
|
||||
Map OpenAI-style tools/functions to Vertex AI format.
|
||||
|
||||
Args:
|
||||
value: List of tool definitions
|
||||
optional_params: Request-scoped parameters to store retrieval config
|
||||
send_integer_enums: Whether to send integer enums with string values
|
||||
|
||||
Returns:
|
||||
List of mapped tools in Vertex AI format
|
||||
|
|
@ -711,18 +713,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
googleSearch: dict | None = None
|
||||
googleSearchRetrieval: dict | None = None
|
||||
enterpriseWebSearch: dict | None = None
|
||||
urlContext: dict | None = None
|
||||
urlContext: dict[str, object] | None = None
|
||||
code_execution: dict | None = None
|
||||
googleMaps: dict | None = None
|
||||
google_maps_retrieval_config: dict | None = None
|
||||
computerUse: dict | None = None
|
||||
# remove 'additionalProperties' from tools
|
||||
schema_value: Final[JsonValue] = cast(JsonValue, value) # cast-ok: tool schemas come from JSON request data
|
||||
schema_value: Final[JsonValue] = deepcopy(cast(JsonValue, value)) # cast-ok: tools are JSON request data
|
||||
remove_additional_properties(schema_value)
|
||||
# remove 'strict' from tools
|
||||
remove_strict_from_schema(schema_value)
|
||||
|
||||
for tool in value:
|
||||
for tool in cast(list[dict], schema_value): # cast-ok: a deep copy keeps the caller's tool shape
|
||||
openai_function_object: ChatCompletionToolParamFunctionChunk | None = None
|
||||
if "function" in tool: # tools list
|
||||
_openai_function_object = ChatCompletionToolParamFunctionChunk(**tool["function"])
|
||||
|
|
@ -732,7 +734,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
and _openai_function_object["parameters"] is not None
|
||||
and isinstance(_openai_function_object["parameters"], dict)
|
||||
): # OPENAI accepts JSON Schema, Google accepts OpenAPI schema.
|
||||
_openai_function_object["parameters"] = build_vertex_schema(_openai_function_object["parameters"])
|
||||
_openai_function_object["parameters"] = build_vertex_schema(
|
||||
_openai_function_object["parameters"], send_integer_enums=send_integer_enums
|
||||
)
|
||||
|
||||
openai_function_object = _openai_function_object
|
||||
|
||||
|
|
@ -875,21 +879,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
self,
|
||||
value: list[dict[str, object]], # mutable-ok: mirrors override contract
|
||||
optional_params: dict[str, object], # mutable-ok: mirrors override contract
|
||||
send_integer_enums: bool = False,
|
||||
) -> list[Tools]: # mutable-ok: mirrors override contract
|
||||
return self._map_function(value, optional_params)
|
||||
return self._map_function(value, optional_params, send_integer_enums)
|
||||
|
||||
def _map_response_schema(self, value: dict) -> dict:
|
||||
def _map_response_schema(self, value: dict, send_integer_enums: bool = False) -> Mapping[str, object]:
|
||||
old_schema = deepcopy(value)
|
||||
if isinstance(old_schema, list):
|
||||
for item in old_schema:
|
||||
if isinstance(item, dict):
|
||||
item = build_vertex_schema(parameters=item, add_property_ordering=True)
|
||||
item = build_vertex_schema(
|
||||
parameters=item, add_property_ordering=True, send_integer_enums=send_integer_enums
|
||||
)
|
||||
|
||||
elif isinstance(old_schema, dict):
|
||||
old_schema = build_vertex_schema(parameters=old_schema, add_property_ordering=True)
|
||||
old_schema = build_vertex_schema(
|
||||
parameters=old_schema, add_property_ordering=True, send_integer_enums=send_integer_enums
|
||||
)
|
||||
return old_schema
|
||||
|
||||
def apply_response_schema_transformation(self, value: dict, optional_params: dict, model: str):
|
||||
def apply_response_schema_transformation(
|
||||
self, value: dict, optional_params: dict, model: str, allow_integer_enums: bool = True
|
||||
):
|
||||
new_value = deepcopy(value)
|
||||
# remove 'strict' from json schema (not supported by Gemini)
|
||||
new_value = remove_strict_from_schema(new_value)
|
||||
|
|
@ -931,7 +942,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
# - OpenAPI-style format (uppercase types)
|
||||
# - No additionalProperties support
|
||||
# - Requires propertyOrdering
|
||||
optional_params["response_schema"] = self._map_response_schema(value=schema)
|
||||
optional_params["response_schema"] = with_integer_enums_that_fit(
|
||||
partial(self._map_response_schema, schema),
|
||||
allow_integer_enums and self._is_gemini_3_or_newer(model),
|
||||
)
|
||||
|
||||
@property
|
||||
def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
|
||||
|
|
@ -1249,6 +1263,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
allow_integer_enums: bool = True,
|
||||
) -> dict:
|
||||
self._apply_include_server_side_tool_invocations(non_default_params, optional_params)
|
||||
gemini_sampling_params_warned: bool = False
|
||||
|
|
@ -1299,7 +1314,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
elif param == "max_tokens" or param == "max_completion_tokens":
|
||||
optional_params["max_output_tokens"] = value
|
||||
elif param == "response_format" and isinstance(value, dict):
|
||||
self.apply_response_schema_transformation(value=value, optional_params=optional_params, model=model)
|
||||
self.apply_response_schema_transformation(
|
||||
value=value, optional_params=optional_params, model=model, allow_integer_enums=allow_integer_enums
|
||||
)
|
||||
elif param == "frequency_penalty":
|
||||
if self._supports_penalty_parameters(model):
|
||||
optional_params["frequency_penalty"] = value
|
||||
|
|
@ -1312,7 +1329,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
optional_params["logprobs"] = value
|
||||
elif (param == "tools" or param == "functions") and isinstance(value, list) and value:
|
||||
# Pass optional_params so _map_function can add toolConfig if needed
|
||||
mapped_tools = self._map_function(value=value, optional_params=optional_params)
|
||||
mapped_tools = with_integer_enums_that_fit(
|
||||
partial(self._map_function, value, optional_params),
|
||||
allow_integer_enums and self._is_gemini_3_or_newer(model),
|
||||
)
|
||||
optional_params = self._add_tools_to_optional_params(optional_params, mapped_tools)
|
||||
elif param == "tool_choice" and (isinstance(value, str) or isinstance(value, dict)):
|
||||
_tool_choice_value = self.map_tool_choice_values(
|
||||
|
|
|
|||
|
|
@ -400,7 +400,11 @@ class LiteLLM:
|
|||
|
||||
|
||||
class Chat:
|
||||
def __init__(self, params, router_obj: "Router | None"):
|
||||
def __init__(
|
||||
self,
|
||||
params: dict[str, object], # mutable-ok: pops the acompletion flag in place
|
||||
router_obj: "Router | None",
|
||||
):
|
||||
self.params = params
|
||||
if self.params.get("acompletion", False) is True:
|
||||
self.params.pop("acompletion")
|
||||
|
|
@ -480,6 +484,7 @@ async def acompletion(
|
|||
service_tier: str | None = None,
|
||||
store: bool | None = None,
|
||||
prompt_cache_key: str | None = None,
|
||||
prompt_cache_retention: str | None = None,
|
||||
# set api_base, api_version, api_key
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
|
|
@ -657,6 +662,7 @@ async def acompletion(
|
|||
"service_tier": service_tier,
|
||||
"store": store,
|
||||
"prompt_cache_key": prompt_cache_key,
|
||||
"prompt_cache_retention": prompt_cache_retention,
|
||||
"extra_headers": extra_headers,
|
||||
"acompletion": True, # assuming this is a required parameter
|
||||
"thinking": thinking,
|
||||
|
|
@ -5356,6 +5362,7 @@ def completion(
|
|||
service_tier: str | None = None,
|
||||
store: bool | None = None,
|
||||
prompt_cache_key: str | None = None,
|
||||
prompt_cache_retention: str | None = None,
|
||||
# soon to be deprecated params by OpenAI
|
||||
functions: list | None = None,
|
||||
function_call: str | None = None,
|
||||
|
|
@ -5478,6 +5485,7 @@ def completion(
|
|||
service_tier=service_tier,
|
||||
store=store,
|
||||
prompt_cache_key=prompt_cache_key,
|
||||
prompt_cache_retention=prompt_cache_retention,
|
||||
base_url=base_url,
|
||||
api_version=api_version,
|
||||
api_key=api_key,
|
||||
|
|
@ -5843,6 +5851,7 @@ def completion(
|
|||
"service_tier": service_tier,
|
||||
"store": store,
|
||||
"prompt_cache_key": prompt_cache_key,
|
||||
"prompt_cache_retention": prompt_cache_retention,
|
||||
"allowed_openai_params": allowed_openai_params,
|
||||
"base_model": base_model,
|
||||
}
|
||||
|
|
@ -5979,6 +5988,7 @@ def completion(
|
|||
*OPENAI_WIF_KWARGS_KEYS,
|
||||
*OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
|
||||
"github_copilot_auth_type",
|
||||
"github_copilot_host",
|
||||
"github_copilot_user_session",
|
||||
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
|
||||
"fireworks_forward_user_id",
|
||||
|
|
@ -9143,7 +9153,7 @@ async def ahealth_check(
|
|||
|
||||
####### HELPER FUNCTIONS ################
|
||||
## Set verbose to true -> ```litellm.set_verbose = True```
|
||||
def print_verbose(print_statement):
|
||||
def print_verbose(print_statement: object):
|
||||
try:
|
||||
verbose_logger.debug(print_statement)
|
||||
if litellm.set_verbose:
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
import inspect
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
|
||||
|
||||
from litellm.exceptions import BadRequestError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.anthropic.pass_through.messages import handler as main
|
||||
from litellm.llms.anthropic.pass_through.safeguards import native_route_classifies
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext
|
||||
from litellm.rust_bridge import detectors
|
||||
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
|
||||
from litellm.rust_bridge.dispatch import PublicDispatch
|
||||
from litellm.rust_bridge.messages import detectors as route_detectors
|
||||
from litellm.rust_bridge.messages.entrypoints import (
|
||||
NATIVE_AMESSAGES,
|
||||
NATIVE_MESSAGES,
|
||||
)
|
||||
from litellm.rust_bridge.public_call import (
|
||||
AsyncFlag,
|
||||
NativeCall,
|
||||
bind,
|
||||
native_call,
|
||||
|
|
@ -46,6 +49,7 @@ def _python_amessages() -> PythonAmessages:
|
|||
|
||||
_PYTHON_MESSAGES: Final = _python_messages()
|
||||
_MESSAGES: Final = signature(_PYTHON_MESSAGES)
|
||||
_FLAG: Final = AsyncFlag("is_async", sync=_MESSAGES, is_set=bool)
|
||||
_PYTHON_AMESSAGES: Final = _python_amessages()
|
||||
_AMESSAGES: Final = signature(_PYTHON_AMESSAGES)
|
||||
|
||||
|
|
@ -71,45 +75,76 @@ def _public_request(
|
|||
return native_call(legacy, args, _native_kwargs(fields, kwargs))
|
||||
|
||||
|
||||
def _resolved_provider(request: NativeCall) -> str | None:
|
||||
try:
|
||||
return get_llm_provider(
|
||||
str(request.resolved["model"]), optional_str(request.resolved.get("custom_llm_provider"))
|
||||
)[1]
|
||||
except BadRequestError:
|
||||
return optional_str(request.resolved.get("custom_llm_provider"))
|
||||
|
||||
|
||||
def _python_serves(request: NativeCall) -> bool:
|
||||
"""Vertex AI serves only Claude through the Anthropic Messages config, as
|
||||
``get_provider_anthropic_messages_config`` does; every other Vertex model keeps Python's
|
||||
translation through Chat Completions. A request Python would answer with classifier
|
||||
``safeguard_results`` stays on Python too."""
|
||||
def _resolved(request: NativeCall) -> tuple[str, str | None]:
|
||||
model: Final = str(request.resolved["model"])
|
||||
vertex_without_claude: Final = _resolved_provider(request) == "vertex_ai" and "claude" not in model.lower()
|
||||
return vertex_without_claude or native_route_classifies(model, request.kwargs.get("safeguards"))
|
||||
provider: Final = optional_str(request.resolved.get("custom_llm_provider"))
|
||||
try:
|
||||
resolved_model, resolved_provider, _, _ = get_llm_provider(model, provider)
|
||||
except BadRequestError:
|
||||
return model, provider
|
||||
return resolved_model, resolved_provider
|
||||
|
||||
|
||||
def _context(request: NativeCall) -> RouteContext:
|
||||
def _target(request: NativeCall, model: str, provider: str | None) -> Target:
|
||||
requested_model: Final = str(request.resolved["model"])
|
||||
model_info: Final = request.resolved.get("model_info")
|
||||
api_base: Final = request.resolved.get("api_base")
|
||||
native_config: Final = main._native_messages_config( # pyright: ignore[reportPrivateUsage] # the handler's own passthrough lookup
|
||||
model, provider, model_info, api_base if isinstance(api_base, str) else None
|
||||
)
|
||||
if native_config is not None:
|
||||
return Target.MESSAGES
|
||||
compacting: Final = request.resolved.get("compaction") is not None
|
||||
if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice
|
||||
return Target.RESPONSES
|
||||
return Target.CHAT_COMPLETIONS
|
||||
|
||||
|
||||
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
||||
{
|
||||
**detectors.COMMON,
|
||||
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
|
||||
Feature.RESPONSE_CACHE: detectors.response_cache,
|
||||
Feature.MODIFY_PARAMS: detectors.modify_params,
|
||||
Feature.WORKLOAD_IDENTITY: route_detectors.anthropic_workload_identity,
|
||||
Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection,
|
||||
Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection,
|
||||
Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry,
|
||||
}
|
||||
)
|
||||
|
||||
_ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
||||
{
|
||||
**_FEATURES,
|
||||
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
|
||||
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
|
||||
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _context(request: NativeCall, features: detectors.Detectors) -> RouteContext:
|
||||
requested_model: Final = str(request.resolved["model"])
|
||||
model, provider = _resolved(request)
|
||||
return RouteContext(
|
||||
Route.MESSAGES,
|
||||
provider=_resolved_provider(request),
|
||||
model=str(request.resolved["model"]),
|
||||
provider=provider,
|
||||
model=requested_model,
|
||||
target=_target(request, model, provider),
|
||||
features=detectors.detected(features, request),
|
||||
)
|
||||
|
||||
|
||||
_DISPATCH: Final = PublicDispatch(
|
||||
route=Route.MESSAGES,
|
||||
request=lambda args, kwargs: _public_request(_MESSAGES, args, kwargs),
|
||||
context=_context,
|
||||
bypass=lambda request: request.kwargs.get("is_async") is True or _python_serves(request),
|
||||
context=lambda request: _context(request, _FEATURES),
|
||||
)
|
||||
|
||||
_ADISPATCH: Final = PublicDispatch(
|
||||
route=Route.MESSAGES,
|
||||
request=lambda args, kwargs: _public_request(_AMESSAGES, args, kwargs),
|
||||
context=_context,
|
||||
bypass=_python_serves,
|
||||
context=lambda request: _context(request, _ASYNC_FEATURES),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -118,9 +153,11 @@ def anthropic_messages_handler(
|
|||
**kwargs: object, # kwargs-ok: preserve the public Anthropic Messages call shape
|
||||
) -> MessagesResult | Coroutine[object, object, MessagesResult]:
|
||||
python: Final = _PYTHON_MESSAGES
|
||||
if _FLAG.requested(kwargs):
|
||||
return anthropic_messages(**_FLAG.keywords(args, kwargs))
|
||||
return _DISPATCH.run(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=python,
|
||||
binding=NATIVE_MESSAGES,
|
||||
native=native_call_hook,
|
||||
|
|
@ -131,7 +168,7 @@ async def anthropic_messages(*args: object, **kwargs: object) -> MessagesResult:
|
|||
python: Final = _PYTHON_AMESSAGES
|
||||
return await _ADISPATCH.arun(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=python,
|
||||
binding=NATIVE_AMESSAGES,
|
||||
native=native_call_hook,
|
||||
|
|
|
|||
|
|
@ -28948,6 +28948,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token_batches": 5e-07,
|
||||
"input_cost_per_token_batches": 1.5e-07,
|
||||
|
|
@ -29062,6 +29063,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3-pro-image-preview": {
|
||||
|
|
@ -29114,6 +29116,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3.1-flash-image": {
|
||||
|
|
@ -29164,6 +29167,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3.1-flash-image-preview": {
|
||||
|
|
@ -29207,6 +29211,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini-3.1-flash-lite-image": {
|
||||
|
|
@ -29302,6 +29307,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -29363,6 +29369,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07
|
||||
|
|
@ -29422,6 +29429,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -29508,6 +29516,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token_batches": 5e-08,
|
||||
"input_cost_per_token_batches": 5e-08,
|
||||
|
|
@ -29562,6 +29571,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"supports_image_size": false
|
||||
},
|
||||
|
|
@ -29608,6 +29618,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"supports_image_size": false
|
||||
},
|
||||
|
|
@ -29795,6 +29806,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
|
||||
"input_cost_per_token_batches": 6.25e-07,
|
||||
|
|
@ -29832,6 +29844,7 @@
|
|||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"prompt_cache_min_tokens": 4096.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -29912,6 +29925,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -29968,6 +29982,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"vertex_ai/gemini-3-flash-preview": {
|
||||
|
|
@ -29992,6 +30007,7 @@
|
|||
"output_cost_per_token_flex": 1.5e-06,
|
||||
"output_cost_per_token_priority": 5.4e-06,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -30074,6 +30090,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
|
|
@ -30138,6 +30155,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -30198,6 +30216,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -30224,6 +30243,7 @@
|
|||
"prompt_cache_min_tokens": 4096.0,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -30369,6 +30389,7 @@
|
|||
"output_cost_per_token_priority": 2.16e-05,
|
||||
"prompt_cache_min_tokens": 4096.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -30455,6 +30476,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -30782,6 +30804,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"supports_reasoning": false
|
||||
},
|
||||
|
|
@ -30826,6 +30849,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query"
|
||||
},
|
||||
"gemini/gemini-3.1-flash-lite-preview": {
|
||||
|
|
@ -30887,6 +30911,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07
|
||||
|
|
@ -31036,6 +31061,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token_batches": 5e-07,
|
||||
"input_cost_per_audio_token_priority": 1.8e-06,
|
||||
|
|
@ -31148,6 +31174,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"supports_reasoning": true
|
||||
},
|
||||
|
|
@ -31212,6 +31239,7 @@
|
|||
"output_cost_per_token_batches": 1.5e-06,
|
||||
"rpm": 1000.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://ai.google.dev/gemini-api/docs/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -31362,6 +31390,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_audio_token_batches": 1.5e-07,
|
||||
"input_cost_per_audio_token_priority": 5.4e-07,
|
||||
|
|
@ -31567,6 +31596,7 @@
|
|||
"search_context_size_medium": 0.035,
|
||||
"search_context_size_high": 0.035
|
||||
},
|
||||
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
|
||||
"google_maps_grounding_cost_per_query": 0.025,
|
||||
"input_cost_per_token_batches": 6.25e-07,
|
||||
"input_cost_per_token_flex": 6.25e-07,
|
||||
|
|
@ -31664,6 +31694,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07,
|
||||
|
|
@ -31725,6 +31756,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -31781,6 +31813,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 5e-07,
|
||||
|
|
@ -31844,6 +31877,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
|
|
@ -31909,6 +31943,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -31970,6 +32005,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -32031,6 +32067,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -32126,6 +32163,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
|
|
@ -32189,6 +32227,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token_flex": 1e-06,
|
||||
|
|
@ -32217,6 +32256,7 @@
|
|||
"output_cost_per_token_flex": 1.5e-06,
|
||||
"output_cost_per_token_priority": 5.4e-06,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -32331,6 +32371,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
|
|
@ -32395,6 +32436,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -32455,6 +32497,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -32480,6 +32523,7 @@
|
|||
"output_cost_per_token_priority": 6.75e-06,
|
||||
"prompt_cache_min_tokens": 4096.0,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_endpoints": [
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -52449,6 +52493,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -52511,6 +52556,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014,
|
||||
"input_cost_per_audio_token_batches": 2.5e-07
|
||||
|
|
@ -52571,6 +52617,7 @@
|
|||
"search_context_size_medium": 0.014,
|
||||
"search_context_size_high": 0.014
|
||||
},
|
||||
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
|
||||
"web_search_billing_unit": "per_query",
|
||||
"google_maps_grounding_cost_per_query": 0.014
|
||||
},
|
||||
|
|
@ -67245,6 +67292,7 @@
|
|||
"fireworks_ai/accounts/fireworks/routers/glm-5p3-flash-us": {
|
||||
"cache_read_input_token_cost": 4.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.625e-08,
|
||||
"deprecation_date": "2026-11-07",
|
||||
"input_cost_per_token": 2.25e-07,
|
||||
"input_cost_per_token_priority": 2.8125e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
|
|
@ -67279,6 +67327,7 @@
|
|||
"fireworks_ai/glm-5p3-flash-us": {
|
||||
"cache_read_input_token_cost": 4.5e-08,
|
||||
"cache_read_input_token_cost_priority": 5.625e-08,
|
||||
"deprecation_date": "2026-11-07",
|
||||
"input_cost_per_token": 2.25e-07,
|
||||
"input_cost_per_token_priority": 2.8125e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
|
|
@ -75015,12 +75064,10 @@
|
|||
"gemini-nano-banana-2.1": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"input_cost_per_image": 0.00168,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"input_cost_per_token_priority": 2.7e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -75031,7 +75078,6 @@
|
|||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_flex": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -75064,12 +75110,10 @@
|
|||
"vertex_ai/gemini-nano-banana-2.1": {
|
||||
"cache_read_input_token_cost": 1.5e-07,
|
||||
"cache_read_input_token_cost_batches": 7.5e-08,
|
||||
"cache_read_input_token_cost_flex": 7.5e-08,
|
||||
"cache_read_input_token_cost_priority": 2.7e-07,
|
||||
"input_cost_per_image": 0.00168,
|
||||
"input_cost_per_token": 1.5e-06,
|
||||
"input_cost_per_token_batches": 7.5e-07,
|
||||
"input_cost_per_token_flex": 7.5e-07,
|
||||
"input_cost_per_token_priority": 2.7e-06,
|
||||
"litellm_provider": "vertex_ai-language-models",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -75080,7 +75124,6 @@
|
|||
"output_cost_per_image_token": 3e-05,
|
||||
"output_cost_per_token": 7.5e-06,
|
||||
"output_cost_per_token_batches": 3.75e-06,
|
||||
"output_cost_per_token_flex": 3.75e-06,
|
||||
"output_cost_per_token_priority": 1.35e-05,
|
||||
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
|
|
@ -76880,5 +76923,25 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
]
|
||||
},
|
||||
"fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash-1017-us": {
|
||||
"cache_read_input_token_cost": 9e-09,
|
||||
"cache_read_input_token_cost_priority": 1.125e-08,
|
||||
"input_cost_per_token": 4.5e-07,
|
||||
"input_cost_per_token_priority": 5.625e-07,
|
||||
"litellm_provider": "fireworks_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_output_tokens": 393216,
|
||||
"max_tokens": 393216,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.8e-06,
|
||||
"output_cost_per_token_priority": 2.25e-06,
|
||||
"source": "https://docs.fireworks.ai/serverless/us-only-serverless",
|
||||
"supports_function_calling": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from typing import Literal, TypeAlias
|
|||
|
||||
from pydantic import ConfigDict, Field, model_validator
|
||||
|
||||
from litellm.constants import GITHUB_COPILOT_DEFAULT_HOST
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
CredentialSource: TypeAlias = Literal["db", "config"]
|
||||
|
|
@ -101,6 +102,7 @@ class UserConnectionFlowHandle(LiteLLMBaseModel):
|
|||
|
||||
user_id: str
|
||||
credential_name: str
|
||||
github_host: str = GITHUB_COPILOT_DEFAULT_HOST
|
||||
device_code: str = Field(repr=False)
|
||||
interval: int
|
||||
expires_at: float
|
||||
|
|
|
|||
|
|
@ -9,7 +9,14 @@ from litellm.rust_bridge import runtime
|
|||
from litellm.rust_bridge.catalog import Route, RouteContext
|
||||
from litellm.rust_bridge.dispatch import PublicDispatch
|
||||
from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR
|
||||
from litellm.rust_bridge.public_call import NativeCall, native_call, native_call_hook, optional_str, signature
|
||||
from litellm.rust_bridge.public_call import (
|
||||
AsyncFlag,
|
||||
NativeCall,
|
||||
native_call,
|
||||
native_call_hook,
|
||||
optional_str,
|
||||
signature,
|
||||
)
|
||||
|
||||
__all__ = ("aocr", "ocr")
|
||||
|
||||
|
|
@ -39,6 +46,7 @@ def _bind_request(
|
|||
|
||||
|
||||
_OCR: Final = signature(_bind_request)
|
||||
_FLAG: Final = AsyncFlag("aocr", sync=_OCR)
|
||||
|
||||
|
||||
def _public_request(name: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> NativeCall:
|
||||
|
|
@ -59,7 +67,6 @@ _DISPATCH: Final = PublicDispatch(
|
|||
route=Route.OCR,
|
||||
request=lambda args, kwargs: _public_request("ocr", args, kwargs),
|
||||
context=_context,
|
||||
bypass=lambda request: request.kwargs.get("aocr") is True,
|
||||
)
|
||||
|
||||
_ADISPATCH: Final = PublicDispatch(
|
||||
|
|
@ -73,9 +80,11 @@ def ocr(
|
|||
*args: object,
|
||||
**kwargs: object, # kwargs-ok: preserve the public OCR call shape
|
||||
) -> OCRResponse | Coroutine[object, object, OCRResponse]:
|
||||
if _FLAG.requested(kwargs):
|
||||
return aocr(**_FLAG.keywords(args, kwargs))
|
||||
return _DISPATCH.run(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=runtime.NO_PYTHON,
|
||||
binding=NATIVE_OCR,
|
||||
native=native_call_hook,
|
||||
|
|
@ -85,7 +94,7 @@ def ocr(
|
|||
async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: preserve the public OCR call shape
|
||||
return await _ADISPATCH.arun(
|
||||
args,
|
||||
kwargs,
|
||||
_FLAG.dropped(kwargs),
|
||||
python=runtime.NO_PYTHON,
|
||||
binding=NATIVE_AOCR,
|
||||
native=native_call_hook,
|
||||
|
|
|
|||
|
|
@ -437,6 +437,7 @@ def llm_passthrough_route(
|
|||
params: QueryParamTypes | None = None,
|
||||
cookies: CookieTypes | None = None,
|
||||
client: HTTPHandler | AsyncHTTPHandler | None = None,
|
||||
guardrailConfig: object = None,
|
||||
**kwargs,
|
||||
) -> (
|
||||
httpx.Response
|
||||
|
|
@ -547,7 +548,7 @@ def llm_passthrough_route(
|
|||
_request_data: dict | None = data if isinstance(data, dict) else (json if isinstance(json, dict) else None)
|
||||
headers, signed_json_body = provider_config.sign_request(
|
||||
headers=headers,
|
||||
litellm_params=litellm_params_dict,
|
||||
litellm_params={**litellm_params_dict, "guardrailConfig": guardrailConfig},
|
||||
request_data=_request_data,
|
||||
api_base=str(updated_url),
|
||||
model=model,
|
||||
|
|
|
|||
|
|
@ -1197,6 +1197,8 @@ def should_strip_caller_authorization(
|
|||
admission credential — a virtual key, an IdP JWT, or an SSO / OIDC /
|
||||
session token whose ``api_key`` is ``None`` — and must never reach
|
||||
the upstream, so it is stripped regardless of the ``api_key`` value.
|
||||
- **Every other auth-enabled mode**: strip when ``Authorization`` is the
|
||||
credential admission validated.
|
||||
"""
|
||||
if _authorization_is_admission_jwt(raw_headers, user_api_key_auth):
|
||||
return True
|
||||
|
|
@ -1216,7 +1218,21 @@ def should_strip_caller_authorization(
|
|||
mcp_server.auth_type == MCPAuth.oauth2 and mcp_server.delegate_auth_to_upstream
|
||||
)
|
||||
if not (mcp_server.is_oauth_passthrough or is_delegated_oauth):
|
||||
return False
|
||||
from litellm.proxy.auth.user_api_key_auth import is_no_auth_dev_mode
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings, # pyright: ignore[reportUnknownVariableType] # legacy global is declared as a bare dict
|
||||
master_key,
|
||||
)
|
||||
|
||||
proxy_general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(general_settings)
|
||||
admission_ran: Final = not is_no_auth_dev_mode(master_key, proxy_general_settings) or (
|
||||
user_api_key_auth is not None and user_api_key_auth.authenticated_by_custom_auth
|
||||
)
|
||||
return admission_ran and (
|
||||
_authorization_is_litellm_admission_credential_with_settings(
|
||||
raw_headers, user_api_key_auth, proxy_general_settings
|
||||
)
|
||||
)
|
||||
|
||||
has_explicit_litellm_admission_header: Final = _has_explicit_litellm_admission_header(raw_headers)
|
||||
if is_delegated_oauth:
|
||||
|
|
@ -1243,16 +1259,39 @@ def _has_explicit_litellm_admission_header(raw_headers: Mapping[str, str] | None
|
|||
return bool(_raw_header_value(raw_headers, "x-litellm-api-key"))
|
||||
|
||||
|
||||
_GENERAL_SETTINGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _authorization_is_litellm_admission_credential(
|
||||
raw_headers: Mapping[str, str] | None,
|
||||
user_api_key_auth: UserAPIKeyAuth | None,
|
||||
) -> bool:
|
||||
from litellm.proxy.proxy_server import (
|
||||
general_settings, # pyright: ignore[reportUnknownVariableType] # legacy global is declared as a bare dict
|
||||
)
|
||||
|
||||
proxy_general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(general_settings)
|
||||
return _authorization_is_litellm_admission_credential_with_settings(
|
||||
raw_headers, user_api_key_auth, proxy_general_settings
|
||||
)
|
||||
|
||||
|
||||
def _authorization_is_litellm_admission_credential_with_settings(
|
||||
raw_headers: Mapping[str, str] | None,
|
||||
user_api_key_auth: UserAPIKeyAuth | None,
|
||||
proxy_general_settings: Mapping[str, object],
|
||||
) -> bool:
|
||||
"""True when ``Authorization`` carries the LiteLLM key admission validated.
|
||||
|
||||
That is the case when no usable ``x-litellm-api-key`` was sent, or when the client repeated the
|
||||
same key in both headers.
|
||||
That is the case when no usable admission header was sent, or when the client repeated the same key in both headers.
|
||||
"""
|
||||
admission_header: Final = _raw_header_value(raw_headers, "x-litellm-api-key")
|
||||
custom_header_name: Final[object | None] = proxy_general_settings.get("litellm_key_header_name")
|
||||
custom_admission_header: Final = (
|
||||
_raw_header_value(raw_headers, custom_header_name.lower())
|
||||
if isinstance(custom_header_name, str) and custom_header_name
|
||||
else None
|
||||
)
|
||||
admission_header: Final = custom_admission_header or _raw_header_value(raw_headers, "x-litellm-api-key")
|
||||
authorization: Final = _raw_header_value(raw_headers, "authorization")
|
||||
if admission_header and authorization:
|
||||
return strip_auth_scheme(authorization, "Bearer") == strip_auth_scheme(admission_header, "Bearer")
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,4 +1,4 @@
|
|||
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3146e697tym4_.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/0ckv5j_fni3xm.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/06yuul4yvgbd0.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
||||
0:{"tree":{"name":"","param":null,"prefetchHints":4176,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":4192,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":4256,"slots":null}}}}},"staleTime":300,"buildId":"kXnLzJ6ylsRPmgSkCkCKM"}
|
||||
0:{"tree":{"name":"","param":null,"prefetchHints":4176,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":4192,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":4256,"slots":null}}}}},"staleTime":300,"buildId":"u_MWVLx_WA9i-AM2JdCsp"}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,516430,t=>{"use strict";var a=t.i(180127);t.s(["ArrowLeftIcon",()=>a.default])},339402,t=>{"use strict";let a=(0,t.i(475254).default)("code",[["path",{d:"m16 18 6-6-6-6",key:"eg8j8"}],["path",{d:"m8 6-6 6 6 6",key:"ppft3o"}]]);t.s(["default",0,a])},837007,t=>{"use strict";var a=t.i(603908);t.s(["PlusIcon",()=>a.default])},440987,t=>{"use strict";var a=t.i(903446);t.s(["SettingsIcon",()=>a.default])},569074,t=>{"use strict";let a=(0,t.i(475254).default)("upload",[["path",{d:"M12 3v12",key:"1x0j5s"}],["path",{d:"m17 8-5-5-5 5",key:"7q97r8"}],["path",{d:"M21 15v4a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2v-4",key:"ih7n3h"}]]);t.s(["Upload",0,a],569074)},868499,t=>{"use strict";var a=t.i(843476);t.s([],558762),t.i(558762);var e=t.i(366250),o=t.i(402820),r=t.i(156736),l=t.i(209793),i=t.i(784324),s=t.i(264951),n=t.i(77173);let d=t.i(313488).DialogTrigger;var u=t.i(974217),g=t.i(325326),c=t.i(301807);let p={modal:!0,disablePointerDismissal:!0,role:"alertdialog"};class f extends g.DialogHandle{constructor(t){super(t??new c.DialogStore(p)),t&&this.store.update(p)}}t.s(["Backdrop",()=>o.DialogBackdrop,"Close",()=>r.DialogClose,"Description",()=>l.DialogDescription,"Handle",0,f,"Popup",()=>i.DialogPopup,"Portal",()=>s.DialogPortal,"Root",0,function(t){return(0,e.useRenderDialogRoot)(t,"alert-dialog")},"Title",()=>n.DialogTitle,"Trigger",0,d,"Viewport",()=>u.DialogViewport,"createHandle",0,function(){return new f}],734604);var m=t.i(734604),m=m,x=t.i(196631),D=t.i(519455);function j({...t}){return(0,a.jsx)(m.Portal,{"data-slot":"alert-dialog-portal",...t})}function v({className:t,...e}){return(0,a.jsx)(m.Backdrop,{"data-slot":"alert-dialog-overlay",className:(0,x.cn)("fixed inset-0 isolate z-popup bg-black/10 duration-100 supports-backdrop-filter:backdrop-blur-xs data-open:animate-in data-open:fade-in-0 data-closed:animate-out data-closed:fade-out-0",t),...e})}t.s(["AlertDialog",0,function({...t}){return(0,a.jsx)(m.Root,{"data-slot":"alert-dialog",...t})},"AlertDialogAction",0,function({className:t,variant:e="default",size:o="default",...r}){return(0,a.jsx)(m.Close,{"data-slot":"alert-dialog-action",className:(0,x.cn)(t),render:(0,a.jsx)(D.Button,{variant:e,size:o}),...r})},"AlertDialogCancel",0,function({className:t,variant:e="outline",size:o="default",...r}){return(0,a.jsx)(m.Close,{"data-slot":"alert-dialog-cancel",className:(0,x.cn)(t),render:(0,a.jsx)(D.Button,{variant:e,size:o}),...r})},"AlertDialogContent",0,function({className:t,size:e="default",...o}){return(0,a.jsxs)(j,{children:[(0,a.jsx)(v,{}),(0,a.jsx)(m.Popup,{"data-slot":"alert-dialog-content","data-size":e,className:(0,x.cn)("group/alert-dialog-content fixed top-1/2 left-1/2 z-popup grid w-full -translate-x-1/2 -translate-y-1/2 gap-6 rounded-xl bg-popover p-6 text-popover-foreground ring-1 ring-foreground/10 duration-100 outline-none data-[size=default]:max-w-xs data-[size=sm]:max-w-xs data-[size=default]:sm:max-w-lg data-open:animate-in data-open:fade-in-0 data-open:zoom-in-95 data-closed:animate-out data-closed:fade-out-0 data-closed:zoom-out-95",t),...o})]})},"AlertDialogDescription",0,function({className:t,...e}){return(0,a.jsx)(m.Description,{"data-slot":"alert-dialog-description",className:(0,x.cn)("text-sm text-balance text-muted-foreground md:text-pretty *:[a]:underline *:[a]:underline-offset-3 *:[a]:hover:text-foreground",t),...e})},"AlertDialogFooter",0,function({className:t,...e}){return(0,a.jsx)("div",{"data-slot":"alert-dialog-footer",className:(0,x.cn)("flex flex-col-reverse gap-2 group-data-[size=sm]/alert-dialog-content:grid group-data-[size=sm]/alert-dialog-content:grid-cols-2 sm:flex-row sm:justify-end",t),...e})},"AlertDialogHeader",0,function({className:t,...e}){return(0,a.jsx)("div",{"data-slot":"alert-dialog-header",className:(0,x.cn)("grid grid-rows-[auto_1fr] place-items-center gap-1.5 text-center has-data-[slot=alert-dialog-media]:grid-rows-[auto_auto_1fr] has-data-[slot=alert-dialog-media]:gap-x-6 sm:group-data-[size=default]/alert-dialog-content:place-items-start sm:group-data-[size=default]/alert-dialog-content:text-left sm:group-data-[size=default]/alert-dialog-content:has-data-[slot=alert-dialog-media]:grid-rows-[auto_1fr]",t),...e})},"AlertDialogTitle",0,function({className:t,...e}){return(0,a.jsx)(m.Title,{"data-slot":"alert-dialog-title",className:(0,x.cn)("text-lg font-medium sm:group-data-[size=default]/alert-dialog-content:group-has-data-[slot=alert-dialog-media]/alert-dialog-content:col-start-2",t),...e})},"AlertDialogTrigger",0,function({...t}){return(0,a.jsx)(m.Trigger,{"data-slot":"alert-dialog-trigger",...t})}],868499)},841238,t=>{"use strict";let a=t=>"object"==typeof t&&null!==t;t.s(["streamErrorMessage",0,t=>{if(!a(t)||!t.error)return null;let{error:e}=t;return"string"==typeof e?e:a(e)&&"string"==typeof e.message?e.message:JSON.stringify(e)}])}]);
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -0,0 +1 @@
|
|||
(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,180127,e=>{"use strict";let t=(0,e.i(475254).default)("arrow-left",[["path",{d:"m12 19-7-7 7-7",key:"1l729n"}],["path",{d:"M19 12H5",key:"x3x0zl"}]]);e.s(["default",0,t])},871689,e=>{"use strict";var t=e.i(180127);e.s(["ArrowLeft",()=>t.default])},595468,e=>{"use strict";var t=e.i(123287);e.s(["CheckCircle2",()=>t.default])},541071,e=>{"use strict";var t=e.i(373488);e.s(["MoreHorizontal",()=>t.default])},332102,e=>{"use strict";let t=(0,e.i(475254).default)("inbox",[["polyline",{points:"22 12 16 12 14 15 10 15 8 12 2 12",key:"o97t9d"}],["path",{d:"M5.45 5.11 2 12v6a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2v-6l-3.45-6.89A2 2 0 0 0 16.76 4H7.24a2 2 0 0 0-1.79 1.11z",key:"oot6mr"}]]);e.s(["Inbox",0,t],332102)},878894,e=>{"use strict";var t=e.i(582458);e.s(["AlertTriangle",()=>t.default])},512154,e=>{e.q("/litellm-asset-prefix/_next/static/media/bing.3b9zkaag7urkm.png")},764453,e=>{e.q("/litellm-asset-prefix/_next/static/media/dataforseo.1g2jptyl8rcb1.png")},341367,e=>{e.q("/litellm-asset-prefix/_next/static/media/exa_ai.36h3hrkelbgj-.png")},732731,e=>{e.q("/litellm-asset-prefix/_next/static/media/google_pse.3hii8gkiytuod.png")},601739,e=>{e.q("/litellm-asset-prefix/_next/static/media/nimble.0ors74qocyffr.png")},911676,e=>{e.q("/litellm-asset-prefix/_next/static/media/parallel_ai.0jx5g5pf0u355.png")},692745,e=>{e.q("/litellm-asset-prefix/_next/static/media/perplexity.2zhky1a8ufk3x.png")},380084,e=>{e.q("/litellm-asset-prefix/_next/static/media/tavily.15dorlkyzxydf.png")},450240,e=>{"use strict";var t=e.i(843476),i=e.i(286536),s=e.i(77705),a=e.i(271645),l=e.i(950594);let r=a.forwardRef(({className:e,groupClassName:r,disabled:n,...p},d)=>{let[o,u]=a.useState(!1);return(0,t.jsxs)(l.InputGroup,{className:r,children:[(0,t.jsx)(l.InputGroupInput,{...p,ref:d,type:o?"text":"password",disabled:n,className:e}),(0,t.jsx)(l.InputGroupAddon,{align:"inline-end",children:(0,t.jsx)(l.InputGroupButton,{size:"icon-xs",disabled:n,"aria-label":o?"Hide password":"Show password",onClick:()=>u(e=>!e),children:o?(0,t.jsx)(s.EyeOff,{}):(0,t.jsx)(i.Eye,{})})})]})});r.displayName="PasswordInput",e.s(["PasswordInput",0,r])}]);
|
||||
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue