mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
9ce96c2d34
* feat(logging): add opt-in session_id/trace_id correlation to JSON log records via contextvars Adds two ContextVar instances (session_id_var, trace_id_var) to litellm/_logging.py and two setter functions (set_session_id, set_trace_id). Logging.__init__() now calls both setters after assigning litellm_trace_id so every JSON log record emitted within the async request context carries trace_id and, when provided, session_id — enabling log correlation in Loki, CloudWatch Logs Insights, and other structured-log sinks without any changes to individual log call sites. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * fix(logging): guard session_id/trace_id injection against overwriting caller-supplied extra fields * fix(logging): always reset session_id_var to empty string when no session_id provided * feat: gate request correlation IDs in logs behind request_correlation_in_logs flag * refactor: move correlation ID injection into CorrelationContextFilter * feat(logging): extend request_correlation_in_logs to plaintext logs and StandardLoggingPayload Plaintext log lines (json_logs off) now get the same trace_id/session_id suffix as JSON logs via a new CorrelationPlainFormatter, so the flag has a visible effect regardless of log format. StandardLoggingPayload gets a new independent session_id field, populated from litellm_session_id. trace_id's existing session_id-first fallback is preserved when request_correlation_in_logs is off; with the flag on, an explicit litellm_trace_id now takes priority over litellm_session_id so the two fields carry genuinely independent values. * fix(logging): restore correlation context after nested calls; sanitize correlation ids Addresses two review findings on this PR. CorrelationContextFilter's trace_id/session_id contextvars were set on every Logging.__init__ but never reset, so a nested LiteLLM call sharing the same asyncio Task as an outer request (e.g. a guardrail's own LLM-as-judge call, an MCP sampling call) would leave the outer request's subsequent log lines stamped with the nested call's ids instead of its own. set_trace_id/ set_session_id now return their contextvars.Token, and Logging stores them and resets both once its own success/failure handler actually completes, via a new idempotent _restore_correlation_context() called from all four terminal handlers. set_trace_id/set_session_id also now strip control characters and bound length before storing a caller-controlled trace_id/session_id, since these values can originate from request input (litellm_session_id, x-litellm- trace-id) and get interpolated into plain-text log lines - without this, a caller could embed \r/\n or escape sequences to forge fake log entries. * fix(logging): restore correlation context after nested calls, not before The previous commit called _restore_correlation_context() as the first line of each terminal handler, before that handler's own callback dispatch loop runs. That's backwards: a nested LiteLLM call triggered from within a callback (e.g. a guardrail's own LLM-as-judge call) would then capture the *already-reset* value as its own pre-call baseline, and its own reset would restore to that instead of the true outer value - verified live to still leak. success_handler/async_success_handler/failure_handler/async_failure_handler are now thin wrappers: the original bodies move to _success_handler_body/etc, called inside a try/finally that restores context only once the full body - including any nested calls its own callback dispatch triggers - has actually finished, mirroring proper stack-scoped nesting semantics. * test(logging): cover async_failure_handler's correlation-context restore Codecov flagged the new async_failure_handler wrapper (try/finally around _async_failure_handler_body) as uncovered - the method had no direct test at all before this PR's refactor split it into a wrapper. Adds a test that awaits it directly and asserts both that async_log_failure_event still fires and that _restore_correlation_context() puts the pre-call trace_id/session_id back. * fix(logging): restore correlation context by value, not by contextvars.Token veria-ai correctly flagged that contextvars.Token.reset() only works in the exact Context it was created in, and litellm's async success path (and streaming failure path) dispatch async_success_handler/async_failure_handler via asyncio.create_task and the global logging worker - a different Context than Logging.__init__ ran in. reset_trace_id/reset_session_id silently swallowed the resulting ValueError, so the restore was a no-op for exactly those paths. Verified independently: reproduced the raw contextvars behavior, then confirmed litellm's async success dispatch really does go through asyncio.create_task + GLOBAL_LOGGING_WORKER (litellm/utils.py). Logging now captures the pre-call *value* (not a Token) and restores via a plain set_trace_id()/set_session_id() call, which works regardless of which Task/Context calls it. reset_trace_id/reset_session_id are removed as dead/unreliable code. Added a regression test that spawns __init__ and the restore in different asyncio Tasks - confirmed it fails against the prior Token-based commit and passes here. * fix(logging): restore correlation context in the originating task too Greptile's re-review correctly identified a remaining gap: for a successful acompletion(), async_success_handler is dispatched via asyncio.create_task + the global logging worker into a *different* Task than the one wrapper_async/Logging.__init__ ran in. The prior fix (43c164a) only restored the handler's own (detached, throwaway) Task - it never touched the originating request Task, which keeps this call's trace_id/ session_id set for the rest of its own execution (e.g. nested calls made via the same Task). wrapper()/wrapper_async() in litellm/utils.py now restore the originating Task's correlation context in a finally block once the whole call is done, regardless of what detached logging tasks it spawned along the way. Since the wrapped body rebinds its own `kwargs` local via function_setup(), sharing the dict object doesn't work here; a small mutable holder carries the constructed Logging instance back out to the outer wrapper instead. _restore_correlation_context() is no longer guarded against repeat calls: with value-based (not Token-based) restoration, each distinct Task that calls it needs its own restore to take effect in that Task's own view of the contextvars, so multiple calls (once per Task involved in an attempt) are required, not just tolerated. Added a regression test using mock_response to exercise the real success dispatch path (asyncio.create_task + GLOBAL_LOGGING_WORKER) without a live provider call, asserting the *test's own* (originating) task context is restored after the call - this is exactly the case Greptile flagged and the prior commit didn't cover. * fix(logging): restore correlation context when function_setup itself fails Greptile's 4th finding: if function_setup() constructs Logging() (whose __init__ already mutates trace_id_var/session_id_var) and then raises before returning - e.g. update_environment_variables() throws - the caller's wrapper()/wrapper_async() never receives a logging_obj reference, so its own restore-on-finally never fires. The correlation ids leak into every subsequent log line on that thread/task with no way to clear them. function_setup()'s own except block now restores the context itself in that case, using whatever logging_obj it managed to construct before failing (locals().get(), safe against the earlier failure modes where logging_obj was never assigned at all). Added a regression test that monkeypatches Logging.update_environment_variables to raise after construction, confirmed it fails without this fix (the leaked ids show up directly in the raised exception's own log line) and passes with it. Broader sweep (test_utils.py, test_router.py, test_main_module_header.py, streaming handler tests, plus all logging-specific tests): 722 passed. * fix(logging): don't assume every litellm_logging_obj is a real Logging instance CI caught a real regression from the last commit: tests/test_litellm/llms/xai/test_xai_key_fallback.py injects a minimal FakeLogging stand-in (only implementing update_from_kwargs) as litellm_logging_obj for a narrow realtime-config unit test, bypassing the real Logging class entirely. wrapper()/ wrapper_async()'s finally block and function_setup()'s except block both unconditionally called _restore_correlation_context() on whatever ended up in the holder, which doesn't exist on that stand-in. _restore_correlation_context is new plumbing specific to this PR's feature, not part of any pre-existing stand-in's expected interface, so callers of it can't assume every object playing the litellm_logging_obj role implements it. Added _restore_correlation_context_if_supported(), a small getattr-guarded helper, and used it at all three call sites. * fix(logging): don't restore context too early on setup failure or streaming Two more findings from Greptile's 5th review round. 1. function_setup()'s except block restored correlation context *after* logging the "Error in function_setup" exception, so that diagnostic log line itself was stamped with the doomed call's ids instead of the outer ids - misleading, since the failed call never produces anything else to attribute those ids to. Restore now happens before the log call. 2. wrapper()/wrapper_async() restored the originating task's context as soon as a streaming call returned, before the caller ever starts iterating the CustomStreamWrapper it just got back. Any log lines emitted while iterating (in the same thread/task) incorrectly showed the pre-call ids instead of this call's own ones. The wrapper finally block now skips the restore when the return value is a stream wrapper, deferring to the terminal handler that already fires once the stream is actually assembled/exhausted. Both verified with tests that fail against the prior commit and pass against this one. Broader sweep unchanged at 829 passing. * fix(logging): best-effort correlation cleanup on abandoned streams Greptile's 7th finding: if a caller returns a streaming response and never fully consumes it - stops iterating early, drops the reference, cancels it - the terminal handler that normally restores the originating task's trace_id/session_id never fires, since it only runs once the stream is actually assembled/exhausted. The ids leak into every subsequent log line in that thread/task with no bound. There's no reliable Python hook for "this was abandoned without being closed" - CustomStreamWrapper has no close()/__aexit__/context-manager convention today, and the only automatic option is __del__, whose timing is inherently unpredictable (delayed by cyclic GC, not guaranteed at interpreter shutdown, can run on a different thread). This is a best-effort safety net, not a guarantee, and is documented as such in the docstring. Testing this via real garbage collection proved unreliable in practice: per-chunk logging submits work to a thread pool executor whose worker thread transiently holds its own bound-method reference to the wrapper until that task completes, so refcount doesn't hit zero on a deterministic schedule even with polling. Tests call __del__ directly instead - a plain method, safe to invoke early - which exercises exactly the restore logic real garbage collection would eventually trigger, plus a case confirming a broken logging_obj can never make __del__ raise. * fix(logging): restore consumer's context at every real stream exit point Two more findings from this round. Veria AI: even a *fully consumed* stream never restored the actual consuming thread/task's correlation context. The terminal success dispatch (dispatch_success_handlers via asyncio.create_task for async, or success_handler via the shared executor for sync) only restores whatever detached context it runs in - never the caller's own thread/task that's running the for/async for loop. Same root cause as the wrapper-level fix two rounds ago, just missed for the streaming-completion path. Greptile: explicit aclose() (client disconnect, router fallback aborting a partial stream) closed the underlying stream without restoring correlation context either, since request wrappers intentionally skip restoration for returned streams and no terminal handler runs on this path. Added CustomStreamWrapper._restore_consumer_correlation_context(), called from every point control genuinely returns to the consumer: the final raise StopIteration/StopAsyncIteration on natural exhaustion (both sync branches, both async branches), _handle_stream_fallback_error (the shared choke point for all three failure-raising call sites), and aclose(). __del__ now delegates to the same helper instead of duplicating it. Verified with tests extending the existing streaming-exhaustion cases to assert the consuming context is restored after the loop completes (fails against the prior commit, passes now), plus a dedicated aclose() test. Broader sweep: 832 passing. * fix(logging): don't let a delayed __del__ finalizer clobber a newer active call If an abandoned stream's __del__ fires late (after cyclic GC delay), a different call may have already taken over the correlation contextvars in the same Task/thread. Restoring unconditionally would stomp that active call's trace_id/session_id with the abandoned stream's stale pre-call snapshot. __del__ now only restores when the contextvars still hold the ids this call itself set. * fix(logging): compare sanitized ids in the __del__ ownership guard set_trace_id()/set_session_id() sanitize (strip control chars, bound length) before storing, so the contextvar's value can differ from the raw litellm_trace_id/litellm_session_id. The __del__ ownership guard was comparing against the raw values, so a caller-supplied id containing control characters or exceeding 256 chars would never match, permanently skipping cleanup. Capture what set_trace_id()/set_session_id() actually stored and compare against that instead. * fix(logging): restore consumer context on the synthesized finish_reason chunk Both __next__ and _finalize_completed_stream() have a branch that fires when the underlying stream ends without ever emitting an explicit finish_reason chunk: they synthesize one via finish_reason_handler() and return it. A consumer that stops as soon as it sees finish_reason - a common pattern - never calls __next__()/__anext__() again, so the existing restore in the sent_last_chunk-is-True StopIteration branch never runs for them. The underlying stream is already exhausted at this point regardless of whether the caller keeps iterating, so restoring here is safe. * fix(logging): don't restore correlation context before the caller receives the final chunk The previous fix (5147c69186) restored context immediately before returning the synthesized finish_reason chunk from __next__/_finalize_completed_stream, reasoning that completion_stream was already exhausted. But that chunk is still this call's own data, and the caller's own application-level log statements processing it run in the same synchronous frame right after the return - restoring first made those lines carry the wrong (outer) ids, exactly what wrapper()/wrapper_async() deliberately avoid by not restoring while a stream is being iterated. Revert to not restoring there. A caller that keeps iterating still gets a correct, deterministic restore on its very next __next__()/__anext__() call (completion_stream is exhausted, so that immediately re-raises StopIteration/StopAsyncIteration through the already-restoring branch). A caller that stops right after finish_reason relies on aclose() or the best-effort __del__ guard, same as any other stream the caller doesn't fully exhaust. * refactor(logging): hoist a safely-hoistable function-body import to module top CorrelationContextFilter.filter()'s `import litellm` was a function-body import; verified it can move to module top without a circular-import failure (litellm/__init__.py already imports from litellm._logging before setting request_correlation_in_logs, but a bare `import litellm` only binds the already-in-sys.modules module object - the attribute itself isn't read until filter() actually runs, by which point litellm is fully initialized). * test(logging): move correlation tests into their conventionally-mapped files tests/test_litellm/ mirrors litellm/ in a parallel path. Correlation tests for the Logging class (litellm_logging.py), function_setup/wrapper_async (utils.py), and CustomStreamWrapper (streaming_handler.py) had all landed in test_logging.py, which only maps to litellm/_logging.py itself. Moving each group to its correctly-mapped file: test_litellm_logging.py (Logging class init/restore), test_utils.py (function_setup, wrapper_async), and test_streaming_handler.py (CustomStreamWrapper) in the next commit. test_logging.py keeps only what actually exercises _logging.py's own contextvars/filters/formatters/sanitization. No behavior change - same assertions, same coverage, just relocated. * fix(logging): restore correlation context unconditionally in wrapper()'s sync path Blocking finding from review: a caller-visible correlation feature was silently misattributing one request's logs to a different, unrelated one on the sync/threaded path. wrapper()/wrapper_async() both left trace_id/session_id "open" across a stream's entire iteration so the caller's own log lines while consuming it would carry the right ids. That's safe for wrapper_async(): each async call gets its own asyncio Task with its own copy of the contextvars, and Tasks are never recycled across requests, so a leftover value can only ever affect that one already-abandoned Task. It is not safe for wrapper() (sync): a plain OS thread has no such per-call isolation, and a thread pool's worker threads *are* recycled across unrelated requests. If a sync stream was abandoned (client disconnect, early break, an uncaught exception) without ever being exhausted or closed, nothing restored its contextvars, and a pool could later hand that same thread to a completely different call, which would inherit the abandoned request's ids as its own "pre-call" baseline and then restore back to that poison when it finished - permanently misattributing every subsequent log line on that thread, including its own, to the abandoned request. Strengthening the __del__ finalizer already added for this can't fix it: finalizer timing is exactly what a permanently-reused thread can't rely on. wrapper() now restores unconditionally in its own finally, before a sync stream is ever handed back to the caller. The trade-off: a sync stream consumer's own application-level log statements while iterating no longer automatically carry this call's ids (litellm's own internal per-chunk logging is unaffected, since it's dispatched separately). That's an acceptable cost for eliminating a silent cross-request misattribution bug. wrapper_async() keeps the existing conditional (skip-if-streaming) behavior, justified by the Task-isolation argument above; CustomStreamWrapper's __del__/aclose()/next-iteration restore machinery remains meaningful and necessary there. This also simplifies wrapper()/wrapper_async() back toward their original shape: both previously used a mutable-dict-holder split into a separate _body function to smuggle logging_obj/result out to an outer finally, working around function_setup() rebinding its own local `kwargs`. That restructuring is no longer needed - `logging_obj` (and, for wrapper_async(), `result`) were already function-level locals in scope for a plain try/finally; three of wrapper_async()'s retry-return statements now assign through `result` first so it accurately reflects what's actually returned even on a retry path. Regression test: test_abandoned_sync_stream_does_not_contaminate_a_later_call_on_the_same_thread in test_streaming_handler.py reproduces the exact reported scenario with a real single-worker ThreadPoolExecutor - confirmed it fails with the prior (skip-restore-on-stream) wrapper() and passes with this fix. * refactor(logging): use Mapping instead of bare dict for read-only params _get_standard_logging_payload_trace_id/_session_id only read litellm_params (.get() calls, no mutation) - annotate it as Mapping[str, Any] rather than a bare mutable dict, per the repo's no-mutable-collection-in-annotation rule. * fix(logging): scope request_correlation_in_logs to the async/proxy path only Blocking review finding: wrapper() (the sync entry point) used the same skip-restore-on-stream design as wrapper_async(), but a plain OS thread has no per-call context isolation the way an asyncio Task does, and a thread pool's worker threads are recycled across unrelated requests - an abandoned sync stream could leave its ids stuck on a thread a pool later hands to a completely different request, misattributing that request's logs. A fix existed and was tested (restore unconditionally in wrapper()'s own finally), but it doesn't benefit this feature's primary consumer - the proxy only ever calls the async entry point - and carries sync-specific complexity this PR doesn't need. Scope the feature to async only instead: Logging.__init__() takes a new supports_correlation_logging parameter (default True), threaded down from a new function_setup(..., is_async_call: bool = True) parameter. wrapper() is the one caller that passes is_async_call=False; every other function_setup() call site (wrapper_async(), the router, and proxy/MCP-internal call sites) is already async and keeps the default. With supports_correlation_logging=False, Logging.__init__() never calls set_trace_id()/set_session_id() at all, so a sync call has nothing to leak in the first place. wrapper() reverts to its pre-review shape with no correlation-specific code at all. StandardLoggingPayload's own trace_id/session_id fields are unaffected either way - they're a deterministic per-call read of self.litellm_trace_id/self.litellm_session_id, not ambient contextvar state, so they were never exposed to the cross-request bug. Full sync/direct-SDK support (stamping + its own safe-restore mechanism) is deferred to a follow-up PR; the fix and its regression test already exist in this branch's history at commit9f3a20f4b2and can be resurrected there. Tests: replaced the two wrapper()-level tests with ones proving the new invariant (sync calls, streaming and non-streaming, never touch trace_id_var/session_id_var even when the caller explicitly passes litellm_trace_id/litellm_session_id), and added a direct unit test for the supports_correlation_logging=False gate on Logging.__init__ itself. Verified live: a real proxy (Postgres-backed, real OpenAI calls) shows clean trace_id/session_id isolation across two concurrent sessions with no cross-contamination; a standalone script confirms real sync SDK calls against a real model never touch the correlation contextvars. * feat(logging): fall back to W3C traceparent/baggage for trace_id/session_id request_correlation_in_logs previously only resolved trace_id/session_id from litellm-specific sources: x-litellm-trace-id/x-litellm-session-id headers, a generic x-<vendor>-session-id header, or Anthropic-style metadata.user_id. If none were present, trace_id fell back to an auto-generated UUID unrelated to anything else, and session_id stayed empty - even when the caller already had real distributed-tracing instrumentation sending the actual industry-standard headers for this. Add a fallback to the W3C Trace Context traceparent header (trace-id component) and W3C Baggage header (session.id entry), so a request already carrying real OpenTelemetry trace context correlates litellm's own logs with the same trace in the caller's observability backend (Datadog, Honeycomb, Tempo, etc.) instead of getting an unrelated generated id. Precedence is unchanged for existing sources: explicit litellm headers and the Anthropic metadata path both still win over this new fallback, which only fires when neither found anything. trace_id and session_id are resolved independently here (unlike the existing chain_id mechanism, which uses one shared value for both), since traceparent and baggage are semantically distinct W3C concepts. New helpers _trace_id_from_traceparent/_session_id_from_baggage in litellm_pre_call_utils.py parse the header formats directly (no new dependency - both are simple fixed-width/delimited strings), wired into LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers() only when the corresponding litellm_trace_id/litellm_session_id key isn't already set by the existing paths. Verified live against a real proxy: a bare traceparent header produces a log trace_id exactly matching its trace-id component; a traceparent alongside an explicit x-litellm-trace-id header (different value) produces a log showing the explicit header's value, proving precedence. * fix(logging): reserve trace_id/session_id in JsonFormatter against message-content spoofing JsonFormatter merges keys parsed from the message body before applying extra record attributes, and the extra-attributes loop skips a key that's already present. A caller-controlled log message that happens to parse as JSON/dict with a "trace_id"/"session_id" key (e.g. the proxy logging a raw request-header dict) could therefore make the JSON record carry the attacker-supplied value instead of the real correlation context set via CorrelationContextFilter. trace_id/session_id are now applied from the LogRecord's own attributes after message-content parsing, unconditionally overwriting anything the message body claimed for those two keys. * style(logging): fix import order (ruff I001) in _logging.py and litellm_logging.py - _logging.py: import litellm belongs after the stdlib from-imports, grouped with the other litellm.* imports, not before them. - litellm_logging.py: the refactor to Mapping introduced a second, separate `from collections.abc import Mapping` instead of merging it into the existing `from collections.abc import Callable` import. Caught by the strict-rule budget gate (ruff-strict-budget.json caps I001 at 0 new violations); both auto-fixed with `ruff check --fix --select I001`. * style(logging): freeze mutable-collection constructions flagged by LIT002 Five sites in this PR's diff built a mutable list/dict literal instead of a frozen value: a plain list of optional strings in CorrelationPlainFormatter, a `kwargs or {}` fallback, a `metadata or {}` fallback, two `[...]` candidate orderings, and a `dict(headers)` copy feeding a dict comprehension. Each is build-once/read-only, so this rewrites them as tuples, MappingProxyType, or a plain conditional `.get()` instead of seeding then reading a fresh mutable collection - no behavior change, confirmed by the existing test suite. Caught by the type-discipline budget gate (LIT002 capped at 0 new violations). * fix(logging): reserve trace_id/session_id even when no correlation context is active Live-proxy verification surfaced a gap in the earlier message-content-spoofing fix (7f390a57fc): that fix only overwrites trace_id/session_id from the LogRecord's own attribute, so it does nothing for a log line emitted before CorrelationContextFilter has stamped anything on this record (e.g. the "Request Headers" debug line, which fires before Logging.__init__() runs for the request). On such a record, a caller-supplied header literally named trace_id/session_id still got promoted into the JSON output via the embedded JSON/dict-repr parser, since there was no genuine value to protect. Fixed at the source: trace_id/session_id are now excluded unconditionally from the message-content-parsing promotion step, not just superseded afterward. Verified live against a real proxy - the exact adversarial request (headers literally named trace_id/session_id) no longer leaks into any JSON log record. Added a regression test for this no-active-context variant specifically, confirmed it fails against the prior commit and passes now. Also fixes an unrelated basedpyright regression from an earlier rebase's conflict resolution: litellm/utils.py's `logging_obj` was incorrectly re-annotated `Final` at its second assignment in function_setup() (it's first declared `None` a few lines earlier), which basedpyright correctly rejects. * fix(proxy): stop logging the raw W3C baggage session_id value _session_id_from_baggage() extracts the caller-controlled session.id entry verbatim - it isn't sanitized until set_session_id() runs later in Logging.__init__(). The debug log line for this extraction interpolated the raw value directly, so a caller could embed terminal control characters or ANSI escape sequences that forge/alter plaintext log output for anyone tailing the proxy's logs. Verified live: a baggage header with an embedded ANSI escape reached the terminal as a real, unescaped control sequence before this fix. Drops the value from the log line entirely (the extraction succeeding is enough signal on its own) rather than sanitizing-then-logging, matching veria-ai's suggestion. Added a regression test using caplog that fails against the prior commit and passes now. * fix(logging): restore consumer context only after stream-failure exception mapping _map_anthropic_exception/_map_aleph_alpha_exception synchronously log a debug diagnostic (the raw status code) as part of exception_type()'s mapping. _handle_stream_fallback_error restored the consumer's outer correlation context before calling exception_type(), so that diagnostic log line carried the outer (or empty) trace_id/session_id instead of the failing stream's own - flagged by Greptile. Moved the restore to run after mapping completes, matching the same restore-after-not-before pattern already applied elsewhere in this file for success/finish_reason handling. Added a regression test that captures the correlation context live during a mocked exception_type() call; fails against the prior commit, passes now. * fix(logging): restore consumer context only after aclose()'s stream close completes aclose() restored the consumer's outer correlation context as its first statement, before awaiting the underlying provider stream's own aclose()/ close(). If that close attempt raises, the except branch's debug diagnostic ran under the already-restored outer context instead of the closing stream's own trace_id/session_id - flagged by Greptile, same restore-too-early pattern as the stream-failure fix inf1cf9589d6. Moved the restore to the end of aclose(), after the close attempt (and its diagnostic logging) completes. Added a regression test with a fake stream whose aclose() raises, capturing the correlation context live during the diagnostic log call; fails against the prior commit, passes now. * style(logging): satisfy new strict-lint budgets introduced upstream (Final, ANN401, S110, TRY300, kwargs typing) Rebasing onto litellm_internal_staging pulled in 116 upstream commits that introduced/tightened several lint gates this PR's own code now trips: - LIT010 (every local/module-level variable must be Final): added Final annotations across _logging.py, litellm_logging.py, streaming_handler.py, litellm_pre_call_utils.py, and utils.py. Where a name is genuinely reassigned (logging_obj: starts None, later set to the real object) or branch-assigned, either restructured into a single ternary expression (ordered_candidates) or suppressed with `# rebind-ok: <reason>` matching this repo's documented escape hatch. - LIT011 (parameter mutation): suppressed the two new `data[key] = value` writes in litellm_pre_call_utils.py with `# rebind-ok`, matching the unsuppressed precedent already used for every other `data[...]` write in the same function - `data` is an intentional out-param there. - ANN001/ANN003/ANN202 (missing parameter/return type annotations): fully typed success_handler/_success_handler_body, their async twins, and failure_handler/_failure_handler_body/async variants in litellm_logging.py, plus function_setup in utils.py (added Rules to its existing TYPE_CHECKING block for the rules_obj: Rules annotation). - ANN401 (explicit Any disallowed): suppressed with `# noqa: ANN401` on the handful of genuinely-heterogeneous result/*args/**kwargs parameters, since ordinary suppression is this repo's documented path. - S110 (try/except/pass): added to the existing BLE001 noqa on the one best-effort correlation-cleanup try/except this PR added. - TRY300 (return inside try): moved two `return result` statements into `else:` blocks in the retry-fallback paths this PR's own diff touched. - reportPrivateUsage (basedpyright): renamed the two new StandardLoggingPayloadSetup static methods (get_standard_logging_payload_ trace_id/session_id) to drop their leading underscore, since they're genuinely called from a sibling module-level function in the same file. No behavior change - confirmed by the full existing test suite (819 passed) plus all four lint gates (ruff format, ruff-strict, type-discipline, basedpyright) passing clean. * fix(lint): stop RUF100 flagging noqa suppressions the strict gate needs CI's plain "ruff check" job uses the default ruff.toml, a narrower config than ruff-strict.toml (used only by the strict-rule budget gate). ANN401 and S110 aren't enabled in the default config, so RUF100 (unused-noqa) flagged the `# noqa: ANN401`/`# noqa: ...,S110` suppressions this PR added as pointless under that config, even though they're genuinely needed under ruff-strict.toml. - ANN401: added to ruff.toml's existing `lint.external` list (same mechanism already used for C901/TID251, enforced by the strict gate but not by this config) - these Any usages are genuinely dynamic/forwarded, so the suppression itself is correct and just needed registering. - S110: fixed the underlying code instead of registering another external code - the try/except/pass in CustomStreamWrapper._restore_consumer_correlation_context now logs at debug level on failure (matching the existing best-effort-cleanup pattern in _record_partial_usage_for_failure elsewhere in this file), which satisfies S110's own suggestion directly and needs no suppression at all. Verified against both ruff.toml and ruff-strict.toml directly, plus all three other gates (ruff format, type-discipline, basedpyright) and the full test suite (821 passed). * fix(lint): scope the ANN401 exemption to file level instead of a repo-wide noqa Ruff has no per-line-scoped way to register a noqa code across configs (that requires the default ruff.toml's lint.external list, which is repo-wide in scope even though the noqa itself is per-line). Since ruff does support file-level exemptions via per-file-ignores, and ANN401 only needed exempting in exactly two files, moved the exemption there instead: - ruff-strict.toml: added [lint.per-file-ignores] disabling ANN401 for litellm_logging.py and utils.py specifically, with a comment explaining why (heterogeneous response/forwarded-args parameters with no fitting concrete type - already verified by trying CostResponseTypes and hitting a real basedpyright mismatch). - ruff.toml: reverted the ANN401 entry from lint.external - no longer needed, since there's no `# noqa: ANN401` left anywhere for RUF100 to second-guess. - Removed the now-redundant `# noqa: ANN401` from the 10 affected parameters in both files, keeping the existing kwargs-ok reasons and adding a short inline comment on the `result`/`*args` lines pointing at the ruff-strict.toml exemption for context. Verified against both configs directly (ANN401 clean under ruff-strict.toml for these files, RUF100 clean under the default config), all four gates (ruff format, ruff-strict, type-discipline, basedpyright), and the full test suite (821 passed). * fix(logging): redact credential-shaped trace_id/session_id before stamping log records CorrelationContextFilter stamps trace_id/session_id onto a LogRecord after SecretRedactionFilter has already run, so a caller-controlled value (e.g. via x-litellm-trace-id or a W3C baggage header) that happens to look like a real credential reached JSON and plaintext logs unredacted. Apply the same credential redaction already used elsewhere in this module at _sanitize_correlation_id(), the single choke point both set_trace_id() and set_session_id() route through, so every caller-facing entry point is covered without depending on filter ordering. * fix(logging): restore correlation context when a stream's max-duration timeout fires CustomStreamWrapper.__anext__() called _check_max_streaming_duration() before entering its try block, so the litellm.Timeout it raises bypassed the except Exception -> _handle_stream_fallback_error path entirely, leaking the timed-out stream's own trace_id/session_id into whatever the consumer's task logs next. Move the check inside the try so it flows through the same restoration path every other stream failure already uses. * test(streaming): make dispatch_failure_handlers mock awaitable for the async max-duration test Moving _check_max_streaming_duration() inside __anext__()'s try block (prior commit) means a max-duration Timeout now dispatches failure handlers through the same path every other stream failure already uses, instead of bypassing it entirely. dispatch_failure_handlers is async on the real Logging class; the test's plain MagicMock logging_obj made asyncio.create_task() choke on a non-coroutine return value once that path actually got exercised. --------- Co-authored-by: Deepanshu <deepanshu.lulla@alpha-sense.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
2341 lines
93 KiB
Python
2341 lines
93 KiB
Python
### Hide pydantic namespace conflict warnings globally ###
|
|
from __future__ import annotations
|
|
|
|
import warnings
|
|
|
|
warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
|
|
# Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
|
|
# This warning can accumulate during streaming and cause memory leaks
|
|
warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
|
|
### INIT VARIABLES #########################
|
|
import threading
|
|
import os
|
|
|
|
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
|
|
import dotenv as _dotenv
|
|
|
|
|
|
def _dev_env_hot_reload_enabled() -> bool:
|
|
"""The proxy exports this flag when started with ``--reload``. A reloaded
|
|
worker is a fresh process that inherits the reloader's environment, so an
|
|
edited ``.env`` value stays masked by the stale inherited one unless we
|
|
let the file win; overriding makes the edit take effect on reload."""
|
|
return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True"
|
|
|
|
|
|
if os.getenv("LITELLM_MODE", "DEV") == "DEV":
|
|
_dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
|
|
|
|
from typing import (
|
|
Any,
|
|
Callable,
|
|
Dict,
|
|
Final,
|
|
get_args,
|
|
List,
|
|
Literal,
|
|
Optional,
|
|
overload,
|
|
Tuple,
|
|
Type,
|
|
TYPE_CHECKING,
|
|
Union,
|
|
)
|
|
from litellm.types.integrations.datadog import DatadogInitParams
|
|
from litellm.types.integrations.newrelic import NewRelicInitParams
|
|
from litellm._logging import (
|
|
set_verbose,
|
|
_turn_on_debug,
|
|
verbose_logger,
|
|
json_logs,
|
|
_turn_on_json,
|
|
log_level,
|
|
)
|
|
import re
|
|
from litellm.constants import (
|
|
DEFAULT_BATCH_SIZE,
|
|
DEFAULT_FLUSH_INTERVAL_SECONDS,
|
|
ROUTER_MAX_FALLBACKS,
|
|
DEFAULT_MAX_RETRIES,
|
|
DEFAULT_REPLICATE_POLLING_RETRIES,
|
|
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS,
|
|
LITELLM_CHAT_PROVIDERS,
|
|
HUMANLOOP_PROMPT_CACHE_TTL_SECONDS,
|
|
OPENAI_CHAT_COMPLETION_PARAMS,
|
|
OPENAI_CHAT_COMPLETION_PARAMS as _openai_completion_params, # backwards compatibility
|
|
OPENAI_FINISH_REASONS,
|
|
OPENAI_FINISH_REASONS as _openai_finish_reasons, # backwards compatibility
|
|
openai_compatible_endpoints,
|
|
openai_compatible_providers,
|
|
openai_text_completion_compatible_providers,
|
|
_openai_like_providers,
|
|
replicate_models,
|
|
clarifai_models,
|
|
huggingface_models,
|
|
modelscope_models,
|
|
empower_models,
|
|
together_ai_models,
|
|
baseten_models,
|
|
WANDB_MODELS,
|
|
REPEATED_STREAMING_CHUNK_LIMIT,
|
|
request_timeout,
|
|
request_timeout_explicitly_set as request_timeout_explicitly_set,
|
|
open_ai_embedding_models,
|
|
cohere_embedding_models,
|
|
bedrock_embedding_models,
|
|
known_tokenizer_config,
|
|
BEDROCK_INVOKE_PROVIDERS_LITERAL,
|
|
BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
|
BEDROCK_CONVERSE_MODELS,
|
|
DEFAULT_MAX_TOKENS,
|
|
DEFAULT_SOFT_BUDGET,
|
|
DEFAULT_ALLOWED_FAILS,
|
|
)
|
|
import httpx
|
|
|
|
# register_async_client_cleanup is lazy-loaded and called on first access
|
|
|
|
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
|
|
|
|
|
|
####################################################
|
|
if set_verbose:
|
|
_turn_on_debug()
|
|
####################################################
|
|
### Callbacks /Logging / Success / Failure Handlers #####
|
|
CALLBACK_TYPES = Union[str, Callable, "CustomLogger"] # CustomLogger is lazy-loaded
|
|
input_callback: List[CALLBACK_TYPES] = []
|
|
success_callback: List[CALLBACK_TYPES] = []
|
|
failure_callback: List[CALLBACK_TYPES] = []
|
|
service_callback: List[CALLBACK_TYPES] = []
|
|
audit_log_callbacks: List[CALLBACK_TYPES] = []
|
|
# logging_callback_manager is lazy-loaded via __getattr__
|
|
_custom_logger_compatible_callbacks_literal = Literal[
|
|
"lago",
|
|
"openmeter",
|
|
"logfire",
|
|
"literalai",
|
|
"litellm_agent",
|
|
"dynamic_rate_limiter",
|
|
"dynamic_rate_limiter_v3",
|
|
"langsmith",
|
|
"prometheus",
|
|
"otel",
|
|
"datadog",
|
|
"datadog_metrics",
|
|
"datadog_llm_observability",
|
|
"galileo",
|
|
"braintrust",
|
|
"arize",
|
|
"arize_phoenix",
|
|
"langtrace",
|
|
"gcs_bucket",
|
|
"azure_storage",
|
|
"opik",
|
|
"argilla",
|
|
"mlflow",
|
|
"langfuse",
|
|
"langfuse_otel",
|
|
"weave_otel",
|
|
"pagerduty",
|
|
"humanloop",
|
|
"azure_sentinel",
|
|
"gcs_pubsub",
|
|
"agentops",
|
|
"anthropic_cache_control_hook",
|
|
"generic_api",
|
|
"resend_email",
|
|
"sendgrid_email",
|
|
"smtp_email",
|
|
"deepeval",
|
|
"s3_v2",
|
|
"aws_sqs",
|
|
"vector_store_pre_call_hook",
|
|
"dotprompt",
|
|
"bitbucket",
|
|
"gitlab",
|
|
"cloudzero",
|
|
"focus",
|
|
"mavvrik",
|
|
"vantage",
|
|
"posthog",
|
|
"levo",
|
|
"compression_interception",
|
|
"newrelic",
|
|
]
|
|
cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
|
|
logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
|
|
_known_custom_logger_compatible_callbacks: List = list(get_args(_custom_logger_compatible_callbacks_literal))
|
|
callbacks: List[
|
|
Union[Callable, _custom_logger_compatible_callbacks_literal, "CustomLogger"] # CustomLogger is lazy-loaded
|
|
] = []
|
|
callback_settings: Dict[str, Dict[str, Any]] = {}
|
|
initialized_langfuse_clients: int = 0
|
|
langfuse_default_tags: Optional[List[str]] = None
|
|
langsmith_batch_size: Optional[int] = None
|
|
prometheus_initialize_budget_metrics: Optional[bool] = False
|
|
prometheus_latency_buckets: Optional[List[float]] = None
|
|
require_auth_for_metrics_endpoint: Optional[bool] = True
|
|
argilla_batch_size: Optional[int] = None
|
|
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
|
|
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
|
|
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
|
|
argilla_transformation_object: Optional[Dict[str, Any]] = None
|
|
_async_input_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded
|
|
[]
|
|
) # internal variable - async custom callbacks are routed here.
|
|
_async_success_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded
|
|
[]
|
|
) # internal variable - async custom callbacks are routed here.
|
|
_async_failure_callback: List[Union[str, Callable, "CustomLogger"]] = ( # CustomLogger is lazy-loaded
|
|
[]
|
|
) # internal variable - async custom callbacks are routed here.
|
|
pre_call_rules: List[Callable] = []
|
|
post_call_rules: List[Callable] = []
|
|
turn_off_message_logging: Optional[bool] = False
|
|
standard_logging_payload_excluded_fields: Optional[List[str]] = (
|
|
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
|
|
)
|
|
log_raw_request_response: bool = False
|
|
request_correlation_in_logs: bool = False
|
|
redact_messages_in_exceptions: Optional[bool] = False
|
|
redact_user_api_key_info: Optional[bool] = False
|
|
# When True (default — preserves historical behavior), the Router appends
|
|
# internal config names (model_group, fallback model groups, deployment
|
|
# timeouts, fallback failure details) onto exception messages and surfaces
|
|
# them to clients via ProxyException.message. Set to False if you do NOT
|
|
# want the proxy's internal model_name / fallback wiring visible to clients.
|
|
# Deprecation: planned to flip to False (redact by default) in a future
|
|
# major release; opt in early with `litellm.expose_router_debug_in_errors
|
|
# = False`.
|
|
expose_router_debug_in_errors: bool = True
|
|
filter_invalid_headers: Optional[bool] = False
|
|
add_user_information_to_llm_headers: Optional[bool] = (
|
|
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
|
|
)
|
|
overwrite_user_with_key_hash: bool = (
|
|
False # force the outgoing `user` param to the hashed api key, so providers see a stable, tamper-proof id
|
|
)
|
|
store_audit_logs = False # Enterprise feature, allow users to see audit logs
|
|
skip_system_message_in_guardrail: bool = False
|
|
skip_tool_message_in_guardrail: bool = False
|
|
### end of callbacks #############
|
|
|
|
email: Optional[str] = (
|
|
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
|
)
|
|
token: Optional[str] = (
|
|
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
|
)
|
|
telemetry = True
|
|
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
|
|
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
|
|
modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))
|
|
use_chat_completions_url_for_anthropic_messages: bool = bool(
|
|
os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
|
|
) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API
|
|
# When True, strip the OpenAI-flavored `usage.total_tokens` field that
|
|
# LiteLLM injects into non-streaming /v1/messages responses, bringing the
|
|
# wire response into line with the Anthropic spec (matches the streaming
|
|
# SSE path, which already omits total_tokens). Default False to preserve
|
|
# backward compatibility for clients that read the LiteLLM-shaped
|
|
# `usage.total_tokens` today. Planned to flip to True in a future major
|
|
# release; opt in early via Python:
|
|
# `litellm.strip_anthropic_total_tokens = True`
|
|
# Or via `litellm_settings.strip_anthropic_total_tokens: true` in
|
|
# config.yaml.
|
|
strip_anthropic_total_tokens: bool = False
|
|
anthropic_sse_ping_interval_seconds: float = 15.0
|
|
route_all_chat_openai_to_responses: bool = (
|
|
os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true"
|
|
) # When True, routes all OpenAI /chat/completions requests through the Responses API bridge
|
|
# When True, Gemini/Vertex Live setup is deferred until client `session.update`.
|
|
# Default False preserves historical behavior (auto-send setup on connect).
|
|
gemini_live_defer_setup: bool = os.getenv("LITELLM_GEMINI_LIVE_DEFER_SETUP", "false").lower() == "true"
|
|
use_legacy_interactions_schema: bool = (
|
|
os.getenv("LITELLM_USE_LEGACY_INTERACTIONS_SCHEMA", "false").lower() == "true"
|
|
) # When True, sends Api-Revision: 2026-05-07 to Google so responses use the legacy `outputs`
|
|
# schema instead of the new `steps` schema. Remove this flag after June 8, 2026.
|
|
retry = True
|
|
### AUTH ###
|
|
api_key: Optional[str] = None
|
|
openai_key: Optional[str] = None
|
|
groq_key: Optional[str] = None
|
|
gigachat_key: Optional[str] = None
|
|
xai_key: Optional[str] = None
|
|
databricks_key: Optional[str] = None
|
|
openai_like_key: Optional[str] = None
|
|
azure_key: Optional[str] = None
|
|
anthropic_key: Optional[str] = None
|
|
autorouter_savings_baseline_model: Optional[str] = None
|
|
replicate_key: Optional[str] = None
|
|
bytez_key: Optional[str] = None
|
|
gdc_key: Optional[str] = None
|
|
gdc_api_base: Optional[str] = None
|
|
cohere_key: Optional[str] = None
|
|
infinity_key: Optional[str] = None
|
|
clarifai_key: Optional[str] = None
|
|
maritalk_key: Optional[str] = None
|
|
ai21_key: Optional[str] = None
|
|
ollama_key: Optional[str] = None
|
|
openrouter_key: Optional[str] = None
|
|
datarobot_key: Optional[str] = None
|
|
predibase_key: Optional[str] = None
|
|
huggingface_key: Optional[str] = None
|
|
vertex_project: Optional[str] = None
|
|
vertex_location: Optional[str] = None
|
|
predibase_tenant_id: Optional[str] = None
|
|
togetherai_api_key: Optional[str] = None
|
|
cloudflare_api_key: Optional[str] = None
|
|
vercel_ai_gateway_key: Optional[str] = None
|
|
baseten_key: Optional[str] = None
|
|
llama_api_key: Optional[str] = None
|
|
aleph_alpha_key: Optional[str] = None
|
|
nlp_cloud_key: Optional[str] = None
|
|
novita_api_key: Optional[str] = None
|
|
snowflake_key: Optional[str] = None
|
|
gradient_ai_api_key: Optional[str] = None
|
|
nebius_key: Optional[str] = None
|
|
wandb_key: Optional[str] = None
|
|
heroku_key: Optional[str] = None
|
|
cometapi_key: Optional[str] = None
|
|
ovhcloud_key: Optional[str] = None
|
|
lemonade_key: Optional[str] = None
|
|
sap_service_key: Optional[str] = None
|
|
amazon_nova_api_key: Optional[str] = None
|
|
inception_key: Optional[str] = None
|
|
common_cloud_provider_auth_params: dict = {
|
|
"params": ["project", "region_name", "token"],
|
|
"providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"],
|
|
}
|
|
use_litellm_proxy: bool = False # when True, requests will be sent to the specified litellm proxy endpoint
|
|
use_client: bool = False
|
|
ssl_verify: Union[str, bool] = True
|
|
ssl_security_level: Optional[str] = None
|
|
ssl_certificate: Optional[str] = None
|
|
user_url_validation: bool = True
|
|
user_url_allowed_hosts: List[str] = []
|
|
provider_url_destination_allowed_hosts: List[str] = []
|
|
ssl_ecdh_curve: Optional[str] = None # Set to 'X25519' to disable PQC and improve performance
|
|
disable_streaming_logging: bool = False
|
|
disable_token_counter: bool = False
|
|
disable_add_transform_inline_image_block: bool = False
|
|
disable_add_user_agent_to_request_tags: bool = False
|
|
disable_anthropic_gemini_context_caching_transform: bool = False
|
|
enable_anthropic_prompt_caching: bool = os.getenv("LITELLM_ENABLE_ANTHROPIC_PROMPT_CACHING", "false").lower() == "true"
|
|
_anthropic_prompt_caching_ttl_env: Optional[str] = os.getenv("LITELLM_ANTHROPIC_PROMPT_CACHING_TTL")
|
|
anthropic_prompt_caching_ttl: Optional[Literal["5m", "1h"]] = (
|
|
"1h" if _anthropic_prompt_caching_ttl_env == "1h" else "5m" if _anthropic_prompt_caching_ttl_env == "5m" else None
|
|
)
|
|
disable_vertex_batch_output_transformation: bool = False
|
|
extra_spend_tag_headers: Optional[List[str]] = None
|
|
in_memory_llm_clients_cache: "LLMClientCache"
|
|
safe_memory_mode: bool = False
|
|
enable_azure_ad_token_refresh: Optional[bool] = False
|
|
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
|
|
proxy_auth: Optional[Any] = None
|
|
### DEFAULT AZURE API VERSION ###
|
|
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
|
|
### DEFAULT WATSONX API VERSION ###
|
|
WATSONX_DEFAULT_API_VERSION = "2024-03-13"
|
|
### COHERE EMBEDDINGS DEFAULT TYPE ###
|
|
COHERE_DEFAULT_EMBEDDING_INPUT_TYPE: "COHERE_EMBEDDING_INPUT_TYPES" = "search_document"
|
|
### CREDENTIALS ###
|
|
credential_list: List["CredentialItem"] = []
|
|
### GUARDRAILS ###
|
|
llamaguard_model_name: Optional[str] = None
|
|
openai_moderations_model_name: Optional[str] = None
|
|
presidio_ad_hoc_recognizers: Optional[str] = None
|
|
google_moderation_confidence_threshold: Optional[float] = None
|
|
llamaguard_unsafe_content_categories: Optional[str] = None
|
|
blocked_user_list: Optional[Union[str, List]] = None
|
|
banned_keywords_list: Optional[Union[str, List]] = None
|
|
llm_guard_mode: Literal["all", "key-specific", "request-specific"] = "all"
|
|
guardrail_name_config_map: Dict[str, GuardrailItem] = {}
|
|
include_cost_in_streaming_usage: bool = False
|
|
reasoning_auto_summary: bool = False
|
|
### PROMPTS ####
|
|
from litellm.types.prompts.init_prompts import PromptSpec
|
|
|
|
prompt_name_config_map: Dict[str, PromptSpec] = {}
|
|
|
|
##################
|
|
### PREVIEW FEATURES ###
|
|
enable_preview_features: bool = False
|
|
return_response_headers: bool = False # get response headers from LLM Api providers - example x-remaining-requests,
|
|
enable_json_schema_validation: bool = False
|
|
enable_model_config_credential_overrides: bool = False
|
|
enable_key_alias_format_validation: bool = (
|
|
False # opt-in validation of key_alias format on /key/generate and /key/update
|
|
)
|
|
enable_gemini_default_thinking_level_low: bool = (
|
|
False # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
|
|
)
|
|
####################
|
|
logging: bool = True
|
|
enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
|
|
require_managed_files: bool = False # proxy only - require target_model_names on POST /v1/files
|
|
enable_caching_on_provider_specific_optional_params: bool = (
|
|
False # feature-flag for caching on optional params - e.g. 'top_k'
|
|
)
|
|
caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
|
caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
|
|
cache: Optional["Cache"] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
|
|
default_in_memory_ttl: Optional[float] = None
|
|
default_redis_ttl: Optional[float] = None
|
|
default_redis_batch_cache_expiry: Optional[float] = None
|
|
model_alias_map: Dict[str, str] = {}
|
|
model_group_settings: Optional["ModelGroupSettings"] = None
|
|
max_budget: float = 0.0 # set the max budget across all providers
|
|
budget_duration: Optional[str] = (
|
|
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
|
|
)
|
|
default_soft_budget: float = DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
|
|
budget_exceeded_throttle_percentage: Optional[float] = None
|
|
forward_traceparent_to_llm_provider: bool = False
|
|
|
|
|
|
_current_cost = 0.0 # private variable, used if max budget is set
|
|
error_logs: Dict = {}
|
|
add_function_to_prompt: bool = (
|
|
False # if function calling not supported by api, append function call details to system prompt
|
|
)
|
|
client_session: Optional[httpx.Client] = None
|
|
aclient_session: Optional[httpx.AsyncClient] = None
|
|
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
|
|
model_cost_map_url: str = os.getenv(
|
|
"LITELLM_MODEL_COST_MAP_URL",
|
|
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json",
|
|
)
|
|
blog_posts_url: str = os.getenv(
|
|
"LITELLM_BLOG_POSTS_URL",
|
|
"https://docs.litellm.ai/blog/rss.xml",
|
|
)
|
|
anthropic_beta_headers_url: str = os.getenv(
|
|
"LITELLM_ANTHROPIC_BETA_HEADERS_URL",
|
|
"https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/anthropic_beta_headers_config.json",
|
|
)
|
|
suppress_debug_info: bool = False
|
|
dynamodb_table_name: Optional[str] = None
|
|
s3_callback_params: Optional[Dict] = None
|
|
s3_audit_callback_params: Optional[Dict] = None
|
|
datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
|
|
datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
|
|
newrelic_params: Optional[Union[NewRelicInitParams, Dict]] = None
|
|
aws_sqs_callback_params: Optional[Dict] = None
|
|
generic_logger_headers: Optional[Dict] = None
|
|
default_key_generate_params: Optional[Dict] = None
|
|
default_key_max_budget_alert_emails: Optional[Dict[str, list]] = None
|
|
upperbound_key_generate_params: Optional[LiteLLM_UpperboundKeyGenerateParams] = None
|
|
key_generation_settings: Optional["StandardKeyGenerationConfig"] = None
|
|
default_internal_user_params: Optional[Dict] = None
|
|
default_team_params: Optional[Union[DefaultTeamSSOParams, Dict]] = None
|
|
default_team_settings: Optional[List] = None
|
|
max_user_budget: Optional[float] = None
|
|
default_max_internal_user_budget: Optional[float] = None
|
|
max_internal_user_budget: Optional[float] = None
|
|
max_ui_session_budget: Optional[float] = (
|
|
1.0 # USD budget for each dashboard login session (playground, test connection)
|
|
)
|
|
internal_user_budget_duration: Optional[str] = None
|
|
tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
|
|
max_end_user_budget: Optional[float] = None
|
|
max_end_user_budget_id: Optional[str] = None
|
|
# When True, end-user IDs extracted from requests are validated against
|
|
# LiteLLM_EndUserTable / LiteLLM_UserTable. Values that do not resolve to a
|
|
# known row are dropped before reaching spend logs. Defaults to False for
|
|
# backwards compatibility — arbitrary client-supplied identifiers still
|
|
# pass through unchanged.
|
|
validate_end_user_id_in_db: bool = False
|
|
disable_end_user_cost_tracking: Optional[bool] = None
|
|
disable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
|
|
enable_end_user_cost_tracking_prometheus_only: Optional[bool] = None
|
|
custom_prometheus_metadata_labels: List[str] = []
|
|
custom_prometheus_tags: List[str] = []
|
|
prometheus_metrics_config: Optional[List] = None
|
|
prometheus_exclude_metrics: Optional[List[str]] = None
|
|
prometheus_exclude_labels: Optional[List[str]] = None
|
|
prometheus_emit_stream_label: bool = False
|
|
# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
|
|
# `litellm_proxy_failed_requests_metric`. Off by default to preserve the
|
|
# pre-unification label set so existing dashboards / recording rules keyed on
|
|
# that metric keep matching after upgrade. Enable when downstream consumers
|
|
# are ready to split 429s by source (vendor vs. litellm) and dimension
|
|
# (RPM/TPM/concurrent/budget).
|
|
prometheus_emit_rate_limit_labels: bool = False
|
|
prometheus_user_budget_label_include_email_alias: bool = False
|
|
prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
|
|
prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
|
|
prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
|
|
disable_add_prefix_to_prompt: bool = False # used by anthropic, to disable adding prefix to prompt
|
|
disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
|
|
public_mcp_servers: Optional[List[str]] = None
|
|
public_mcp_hub_strict_whitelist: bool = True
|
|
public_model_groups: Optional[List[str]] = None
|
|
public_agent_groups: Optional[List[str]] = None
|
|
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
|
|
# New format: { "displayName": { "url": "...", "index": 0 } }
|
|
# Old format: { "displayName": "url" } (for backward compatibility)
|
|
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
|
|
#### REQUEST PRIORITIZATION #######
|
|
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
|
|
# priority_reservation_settings is lazy-loaded via __getattr__
|
|
# Only declare for type checking - at runtime __getattr__ handles it
|
|
if TYPE_CHECKING:
|
|
priority_reservation_settings: Optional["PriorityReservationSettings"] = None
|
|
|
|
|
|
######## Networking Settings ########
|
|
use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
|
|
aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
|
|
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
|
|
disable_aiohttp_trust_env: bool = False # When False, aiohttp will respect HTTP(S)_PROXY env vars
|
|
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
|
|
network_mock: bool = False # When True, use mock transport — no real network calls
|
|
|
|
####### STOP SEQUENCE LIMIT #######
|
|
disable_stop_sequence_limit: bool = False # when True, stop sequence limit is disabled
|
|
|
|
#### RETRIES ####
|
|
num_retries: Optional[int] = None # per model endpoint
|
|
max_fallbacks: Optional[int] = None
|
|
default_fallbacks: Optional[List] = None
|
|
fallbacks: Optional[List] = None
|
|
context_window_fallbacks: Optional[List] = None
|
|
content_policy_fallbacks: Optional[List] = None
|
|
allowed_fails: int = 3
|
|
allow_dynamic_callback_disabling: bool = True
|
|
num_retries_per_request: Optional[int] = None # for the request overall (incl. fallbacks + model retries)
|
|
####### SECRET MANAGERS #####################
|
|
secret_manager_client: Optional[Any] = (
|
|
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
|
|
)
|
|
_google_kms_resource_name: Optional[str] = None
|
|
_key_management_system: Optional["KeyManagementSystem"] = None
|
|
# Note: KeyManagementSettings must be eagerly imported because _key_management_settings
|
|
# is accessed during import time in secret_managers/main.py
|
|
# We'll import it after the lazy import system is set up
|
|
# We can't define it here because KeyManagementSettings is lazy-loaded
|
|
#### PII MASKING ####
|
|
output_parse_pii: bool = False
|
|
#############################################
|
|
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
|
|
|
|
model_cost = get_model_cost_map(url=model_cost_map_url)
|
|
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
|
|
cost_margin_config: Dict[
|
|
str, Union[float, Dict[str, float]]
|
|
] = {} # Provider-specific or global cost margins. Examples:
|
|
# Percentage: {"openai": 0.10} = 10% margin
|
|
# Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request
|
|
# Global: {"global": 0.05} = 5% global margin on all providers
|
|
# Combined: {"vertex_ai": {"percentage": 0.08, "fixed_amount": 0.0005}}
|
|
custom_prompt_dict: Dict[str, dict] = {}
|
|
check_provider_endpoint = False
|
|
|
|
|
|
####### THREAD-SPECIFIC DATA ####################
|
|
class MyLocal(threading.local):
|
|
def __init__(self):
|
|
self.user = "Hello World"
|
|
|
|
|
|
_thread_context = MyLocal()
|
|
|
|
|
|
def identify(event_details):
|
|
# Store user in thread local data
|
|
if "user" in event_details:
|
|
_thread_context.user = event_details["user"]
|
|
|
|
|
|
####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc.
|
|
api_base: Optional[str] = None
|
|
headers = None
|
|
api_version: Optional[str] = None
|
|
organization = None
|
|
project = None
|
|
config_path = None
|
|
vertex_ai_safety_settings: Optional[dict] = None
|
|
|
|
####### COMPLETION MODELS ###################
|
|
from typing import Set
|
|
|
|
open_ai_chat_completion_models: Set = set()
|
|
open_ai_text_completion_models: Set = set()
|
|
cohere_models: Set = set()
|
|
cohere_chat_models: Set = set()
|
|
mistral_chat_models: Set = set()
|
|
text_completion_codestral_models: Set = set()
|
|
text_completion_inception_models: Set = set()
|
|
anthropic_models: Set = set()
|
|
openrouter_models: Set = set()
|
|
datarobot_models: Set = set()
|
|
vertex_language_models: Set = set()
|
|
vertex_vision_models: Set = set()
|
|
vertex_chat_models: Set = set()
|
|
vertex_code_chat_models: Set = set()
|
|
vertex_ai_image_models: Set = set()
|
|
vertex_ai_video_models: Set = set()
|
|
vertex_text_models: Set = set()
|
|
vertex_code_text_models: Set = set()
|
|
vertex_embedding_models: Set = set()
|
|
vertex_anthropic_models: Set = set()
|
|
vertex_llama3_models: Set = set()
|
|
vertex_deepseek_models: Set = set()
|
|
vertex_ai_ai21_models: Set = set()
|
|
vertex_mistral_models: Set = set()
|
|
vertex_openai_models: Set = set()
|
|
vertex_minimax_models: Set = set()
|
|
vertex_moonshot_models: Set = set()
|
|
vertex_zai_models: Set = set()
|
|
ai21_models: Set = set()
|
|
ai21_chat_models: Set = set()
|
|
nlp_cloud_models: Set = set()
|
|
aleph_alpha_models: Set = set()
|
|
bedrock_models: Set = set()
|
|
bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
|
|
fal_ai_models: Set = set()
|
|
fireworks_ai_models: Set = set()
|
|
fireworks_ai_embedding_models: Set = set()
|
|
deepinfra_models: Set = set()
|
|
perplexity_models: Set = set()
|
|
watsonx_models: Set = set()
|
|
gemini_models: Set = set()
|
|
xai_models: Set = set()
|
|
zai_models: Set = set()
|
|
deepseek_models: Set = set()
|
|
tencent_models: Set = set()
|
|
runwayml_models: Set = set()
|
|
azure_ai_models: Set = set()
|
|
jina_ai_models: Set = set()
|
|
voyage_models: Set = set()
|
|
infinity_models: Set = set()
|
|
heroku_models: Set = set()
|
|
databricks_models: Set = set()
|
|
cloudflare_models: Set = set()
|
|
codestral_models: Set = set()
|
|
friendliai_models: Set = set()
|
|
featherless_ai_models: Set = set()
|
|
palm_models: Set = set()
|
|
groq_models: Set = set()
|
|
azure_models: Set = set()
|
|
azure_anthropic_models: Set = set()
|
|
azure_text_models: Set = set()
|
|
anyscale_models: Set = set()
|
|
cerebras_models: Set = set()
|
|
galadriel_models: Set = set()
|
|
nvidia_nim_models: Set = set()
|
|
nvidia_riva_models: Set = set()
|
|
soniox_models: Set = set()
|
|
sambanova_models: Set = set()
|
|
sambanova_embedding_models: Set = set()
|
|
novita_models: Set = set()
|
|
assemblyai_models: Set = set()
|
|
snowflake_models: Set = set()
|
|
gradient_ai_models: Set = set()
|
|
llama_models: Set = set()
|
|
nscale_models: Set = set()
|
|
nebius_models: Set = set()
|
|
nebius_embedding_models: Set = set()
|
|
aiml_models: Set = set()
|
|
deepgram_models: Set = set()
|
|
elevenlabs_models: Set = set()
|
|
dashscope_models: Set = set()
|
|
moonshot_models: Set = set()
|
|
publicai_models: Set = set()
|
|
darkbloom_models: Set = set()
|
|
v0_models: Set = set()
|
|
morph_models: Set = set()
|
|
lambda_ai_models: Set = set()
|
|
inception_models: Set = set()
|
|
hyperbolic_models: Set = set()
|
|
black_forest_labs_models: Set = set()
|
|
recraft_models: Set = set()
|
|
cometapi_models: Set = set()
|
|
oci_models: Set = set()
|
|
vercel_ai_gateway_models: Set = set()
|
|
volcengine_models: Set = set()
|
|
wandb_models: Set = set(WANDB_MODELS)
|
|
ovhcloud_models: Set = set()
|
|
ovhcloud_embedding_models: Set = set()
|
|
lemonade_models: Set = set()
|
|
docker_model_runner_models: Set = set()
|
|
amazon_nova_models: Set = set()
|
|
stability_models: Set = set()
|
|
github_copilot_models: Set = set()
|
|
chatgpt_models: Set = set()
|
|
minimax_models: Set = set()
|
|
aws_polly_models: Set = set()
|
|
gigachat_models: Set = set()
|
|
llamagate_models: Set = set()
|
|
reducto_models: Set = set()
|
|
bedrock_mantle_models: Set = set()
|
|
|
|
|
|
def is_bedrock_pricing_only_model(key: str) -> bool:
|
|
"""
|
|
Excludes keys with the pattern 'bedrock/<region>/<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
|
|
|
|
Args:
|
|
key (str): A key to filter.
|
|
|
|
Returns:
|
|
bool: True if the key matches the Bedrock pattern, False otherwise.
|
|
"""
|
|
# Regex to match 'bedrock/<region>/<model>'
|
|
bedrock_pattern: Final = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$")
|
|
|
|
if "month-commitment" in key:
|
|
return True
|
|
|
|
is_match: Final = bedrock_pattern.match(key)
|
|
return is_match is not None
|
|
|
|
|
|
def is_openai_finetune_model(key: str) -> bool:
|
|
"""
|
|
Excludes model cost keys with the pattern 'ft:<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
|
|
|
|
Args:
|
|
key (str): A key to filter.
|
|
|
|
Returns:
|
|
bool: True if the key matches the OpenAI finetune pattern, False otherwise.
|
|
"""
|
|
return key.startswith("ft:") and not key.count(":") > 1
|
|
|
|
|
|
def _populate_provider_model_sets(model_cost_map: Dict) -> None:
|
|
for key, value in model_cost_map.items():
|
|
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
|
|
open_ai_chat_completion_models.add(key)
|
|
elif value.get("litellm_provider") == "text-completion-openai":
|
|
open_ai_text_completion_models.add(key)
|
|
elif value.get("litellm_provider") == "azure_text":
|
|
azure_text_models.add(key)
|
|
elif value.get("litellm_provider") == "cohere":
|
|
cohere_models.add(key)
|
|
elif value.get("litellm_provider") == "cohere_chat":
|
|
cohere_chat_models.add(key)
|
|
elif value.get("litellm_provider") == "mistral":
|
|
mistral_chat_models.add(key)
|
|
elif value.get("litellm_provider") == "anthropic":
|
|
anthropic_models.add(key)
|
|
elif value.get("litellm_provider") == "empower":
|
|
empower_models.add(key)
|
|
elif value.get("litellm_provider") == "openrouter":
|
|
openrouter_models.add(key)
|
|
elif value.get("litellm_provider") == "vercel_ai_gateway":
|
|
vercel_ai_gateway_models.add(key)
|
|
elif value.get("litellm_provider") == "datarobot":
|
|
datarobot_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-text-models":
|
|
vertex_text_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
|
|
vertex_code_text_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-language-models":
|
|
vertex_language_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-vision-models":
|
|
vertex_vision_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-chat-models":
|
|
vertex_chat_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
|
|
vertex_code_chat_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-embedding-models":
|
|
vertex_embedding_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-anthropic_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_anthropic_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-llama_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_llama3_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_deepseek_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_mistral_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-ai21_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_ai_ai21_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-image-models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_ai_image_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-video-models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_ai_video_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-openai_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_openai_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-minimax_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_minimax_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-moonshot_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_moonshot_models.add(key)
|
|
elif value.get("litellm_provider") == "vertex_ai-zai_models":
|
|
key = key.replace("vertex_ai/", "")
|
|
vertex_zai_models.add(key)
|
|
elif value.get("litellm_provider") == "ai21":
|
|
if value.get("mode") == "chat":
|
|
ai21_chat_models.add(key)
|
|
else:
|
|
ai21_models.add(key)
|
|
elif value.get("litellm_provider") == "nlp_cloud":
|
|
nlp_cloud_models.add(key)
|
|
elif value.get("litellm_provider") == "aleph_alpha":
|
|
aleph_alpha_models.add(key)
|
|
elif value.get("litellm_provider") == "bedrock" and not is_bedrock_pricing_only_model(key):
|
|
bedrock_models.add(key)
|
|
elif value.get("litellm_provider") == "bedrock_converse":
|
|
bedrock_converse_models.add(key)
|
|
elif value.get("litellm_provider") == "deepinfra":
|
|
deepinfra_models.add(key)
|
|
elif value.get("litellm_provider") == "perplexity":
|
|
perplexity_models.add(key)
|
|
elif value.get("litellm_provider") == "watsonx":
|
|
watsonx_models.add(key)
|
|
elif value.get("litellm_provider") == "gemini":
|
|
gemini_models.add(key)
|
|
elif value.get("litellm_provider") == "fireworks_ai":
|
|
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
|
if "-to-" not in key and "fireworks-ai-default" not in key:
|
|
fireworks_ai_models.add(key)
|
|
elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
|
|
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
|
if "-to-" not in key:
|
|
fireworks_ai_embedding_models.add(key)
|
|
elif value.get("litellm_provider") == "text-completion-codestral":
|
|
text_completion_codestral_models.add(key)
|
|
elif value.get("litellm_provider") == "text-completion-inception":
|
|
text_completion_inception_models.add(key)
|
|
elif value.get("litellm_provider") == "xai":
|
|
xai_models.add(key)
|
|
elif value.get("litellm_provider") == "zai":
|
|
zai_models.add(key)
|
|
elif value.get("litellm_provider") == "fal_ai":
|
|
fal_ai_models.add(key)
|
|
elif value.get("litellm_provider") == "deepseek":
|
|
deepseek_models.add(key)
|
|
elif value.get("litellm_provider") == "tencent":
|
|
tencent_models.add(key)
|
|
elif value.get("litellm_provider") == "runwayml":
|
|
runwayml_models.add(key)
|
|
elif value.get("litellm_provider") == "meta_llama":
|
|
llama_models.add(key)
|
|
elif value.get("litellm_provider") == "nscale":
|
|
nscale_models.add(key)
|
|
elif value.get("litellm_provider") == "azure_ai":
|
|
azure_ai_models.add(key)
|
|
elif value.get("litellm_provider") == "voyage":
|
|
voyage_models.add(key)
|
|
elif value.get("litellm_provider") == "infinity":
|
|
infinity_models.add(key)
|
|
elif value.get("litellm_provider") == "databricks":
|
|
databricks_models.add(key)
|
|
elif value.get("litellm_provider") == "cloudflare":
|
|
cloudflare_models.add(key)
|
|
elif value.get("litellm_provider") == "codestral":
|
|
codestral_models.add(key)
|
|
elif value.get("litellm_provider") == "friendliai":
|
|
friendliai_models.add(key)
|
|
elif value.get("litellm_provider") == "palm":
|
|
palm_models.add(key)
|
|
elif value.get("litellm_provider") == "groq":
|
|
groq_models.add(key)
|
|
elif value.get("litellm_provider") == "azure":
|
|
azure_models.add(key)
|
|
elif value.get("litellm_provider") == "azure_anthropic":
|
|
azure_anthropic_models.add(key)
|
|
elif value.get("litellm_provider") == "anyscale":
|
|
anyscale_models.add(key)
|
|
elif value.get("litellm_provider") == "cerebras":
|
|
cerebras_models.add(key)
|
|
elif value.get("litellm_provider") == "galadriel":
|
|
galadriel_models.add(key)
|
|
elif value.get("litellm_provider") == "nvidia_nim":
|
|
nvidia_nim_models.add(key)
|
|
elif value.get("litellm_provider") == "nvidia_riva":
|
|
nvidia_riva_models.add(key)
|
|
elif value.get("litellm_provider") == "soniox":
|
|
soniox_models.add(key)
|
|
elif value.get("litellm_provider") == "sambanova":
|
|
sambanova_models.add(key)
|
|
elif value.get("litellm_provider") == "sambanova-embedding-models":
|
|
sambanova_embedding_models.add(key)
|
|
elif value.get("litellm_provider") == "novita":
|
|
novita_models.add(key)
|
|
elif value.get("litellm_provider") == "nebius-chat-models":
|
|
nebius_models.add(key)
|
|
elif value.get("litellm_provider") == "nebius-embedding-models":
|
|
nebius_embedding_models.add(key)
|
|
elif value.get("litellm_provider") == "aiml":
|
|
aiml_models.add(key)
|
|
elif value.get("litellm_provider") == "assemblyai":
|
|
assemblyai_models.add(key)
|
|
elif value.get("litellm_provider") == "jina_ai":
|
|
jina_ai_models.add(key)
|
|
elif value.get("litellm_provider") == "snowflake":
|
|
snowflake_models.add(key)
|
|
elif value.get("litellm_provider") == "gradient_ai":
|
|
gradient_ai_models.add(key)
|
|
elif value.get("litellm_provider") == "featherless_ai":
|
|
featherless_ai_models.add(key)
|
|
elif value.get("litellm_provider") == "deepgram":
|
|
deepgram_models.add(key)
|
|
elif value.get("litellm_provider") == "elevenlabs":
|
|
elevenlabs_models.add(key)
|
|
elif value.get("litellm_provider") == "heroku":
|
|
heroku_models.add(key)
|
|
elif value.get("litellm_provider") == "dashscope":
|
|
dashscope_models.add(key)
|
|
elif value.get("litellm_provider") == "modelscope":
|
|
modelscope_models.add(key)
|
|
elif value.get("litellm_provider") == "moonshot":
|
|
moonshot_models.add(key)
|
|
elif value.get("litellm_provider") == "publicai":
|
|
publicai_models.add(key)
|
|
elif value.get("litellm_provider") == "darkbloom":
|
|
darkbloom_models.add(key)
|
|
elif value.get("litellm_provider") == "v0":
|
|
v0_models.add(key)
|
|
elif value.get("litellm_provider") == "morph":
|
|
morph_models.add(key)
|
|
elif value.get("litellm_provider") == "lambda_ai":
|
|
lambda_ai_models.add(key)
|
|
elif value.get("litellm_provider") == "inception":
|
|
inception_models.add(key)
|
|
elif value.get("litellm_provider") == "hyperbolic":
|
|
hyperbolic_models.add(key)
|
|
elif value.get("litellm_provider") == "black_forest_labs":
|
|
black_forest_labs_models.add(key)
|
|
elif value.get("litellm_provider") == "recraft":
|
|
recraft_models.add(key)
|
|
elif value.get("litellm_provider") == "cometapi":
|
|
cometapi_models.add(key)
|
|
elif value.get("litellm_provider") == "oci":
|
|
oci_models.add(key)
|
|
elif value.get("litellm_provider") == "volcengine":
|
|
volcengine_models.add(key)
|
|
elif value.get("litellm_provider") == "wandb":
|
|
wandb_models.add(key)
|
|
elif value.get("litellm_provider") == "ovhcloud":
|
|
ovhcloud_models.add(key)
|
|
elif value.get("litellm_provider") == "ovhcloud-embedding-models":
|
|
ovhcloud_embedding_models.add(key)
|
|
elif value.get("litellm_provider") == "lemonade":
|
|
lemonade_models.add(key)
|
|
elif value.get("litellm_provider") == "docker_model_runner":
|
|
docker_model_runner_models.add(key)
|
|
elif value.get("litellm_provider") == "amazon_nova":
|
|
amazon_nova_models.add(key)
|
|
elif value.get("litellm_provider") == "stability":
|
|
stability_models.add(key)
|
|
elif value.get("litellm_provider") == "github_copilot":
|
|
github_copilot_models.add(key)
|
|
elif value.get("litellm_provider") == "chatgpt":
|
|
chatgpt_models.add(key)
|
|
elif value.get("litellm_provider") == "minimax":
|
|
minimax_models.add(key)
|
|
elif value.get("litellm_provider") == "aws_polly":
|
|
aws_polly_models.add(key)
|
|
elif value.get("litellm_provider") == "gigachat":
|
|
gigachat_models.add(key)
|
|
elif value.get("litellm_provider") == "llamagate":
|
|
llamagate_models.add(key)
|
|
elif value.get("litellm_provider") == "reducto":
|
|
reducto_models.add(key)
|
|
elif value.get("litellm_provider") == "bedrock_mantle":
|
|
bedrock_mantle_models.add(key)
|
|
|
|
|
|
def add_known_models(model_cost_map: Optional[Dict] = None):
|
|
"""Fold `model_cost_map` (defaults to `litellm.model_cost`) into the per-provider model sets,
|
|
then refresh `models_by_provider` from those sets so the additions reach wildcard expansion.
|
|
The refresh updates the dict in place, so references captured before a reload stay live.
|
|
"""
|
|
_populate_provider_model_sets(model_cost_map if model_cost_map is not None else model_cost)
|
|
models_by_provider.update(_build_models_by_provider())
|
|
|
|
|
|
_populate_provider_model_sets(model_cost)
|
|
# known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary
|
|
|
|
# this is maintained for Exception Mapping
|
|
|
|
|
|
# used for Cost Tracking & Token counting
|
|
# https://azure.microsoft.com/en-in/pricing/details/cognitive-services/openai-service/
|
|
# Azure returns gpt-35-turbo in their responses, we need to map this to azure/gpt-3.5-turbo for token counting
|
|
azure_llms = {
|
|
"gpt-35-turbo": "azure/gpt-35-turbo",
|
|
"gpt-35-turbo-16k": "azure/gpt-35-turbo-16k",
|
|
"gpt-35-turbo-instruct": "azure/gpt-35-turbo-instruct",
|
|
"azure/gpt-41": "gpt-4.1",
|
|
"azure/gpt-41-mini": "gpt-4.1-mini",
|
|
"azure/gpt-41-nano": "gpt-4.1-nano",
|
|
}
|
|
|
|
azure_embedding_models = {
|
|
"ada": "azure/ada",
|
|
}
|
|
|
|
petals_models = [
|
|
"petals-team/StableBeluga2",
|
|
]
|
|
|
|
ollama_models = ["llama2"]
|
|
|
|
maritalk_models = ["maritalk"]
|
|
|
|
model_list = list(
|
|
open_ai_chat_completion_models
|
|
| open_ai_text_completion_models
|
|
| cohere_models
|
|
| cohere_chat_models
|
|
| anthropic_models
|
|
| set(replicate_models)
|
|
| openrouter_models
|
|
| datarobot_models
|
|
| set(huggingface_models)
|
|
| vertex_chat_models
|
|
| vertex_text_models
|
|
| ai21_models
|
|
| ai21_chat_models
|
|
| set(together_ai_models)
|
|
| set(baseten_models)
|
|
| aleph_alpha_models
|
|
| nlp_cloud_models
|
|
| set(ollama_models)
|
|
| bedrock_models
|
|
| deepinfra_models
|
|
| perplexity_models
|
|
| set(maritalk_models)
|
|
| runwayml_models
|
|
| vertex_language_models
|
|
| watsonx_models
|
|
| gemini_models
|
|
| text_completion_codestral_models
|
|
| text_completion_inception_models
|
|
| xai_models
|
|
| zai_models
|
|
| fal_ai_models
|
|
| deepseek_models
|
|
| modelscope_models
|
|
| azure_ai_models
|
|
| voyage_models
|
|
| infinity_models
|
|
| databricks_models
|
|
| cloudflare_models
|
|
| codestral_models
|
|
| friendliai_models
|
|
| palm_models
|
|
| groq_models
|
|
| azure_models
|
|
| azure_anthropic_models
|
|
| anyscale_models
|
|
| cerebras_models
|
|
| galadriel_models
|
|
| nvidia_nim_models
|
|
| nvidia_riva_models
|
|
| soniox_models
|
|
| sambanova_models
|
|
| azure_text_models
|
|
| novita_models
|
|
| assemblyai_models
|
|
| jina_ai_models
|
|
| snowflake_models
|
|
| gradient_ai_models
|
|
| llama_models
|
|
| featherless_ai_models
|
|
| nscale_models
|
|
| deepgram_models
|
|
| elevenlabs_models
|
|
| dashscope_models
|
|
| moonshot_models
|
|
| publicai_models
|
|
| darkbloom_models
|
|
| v0_models
|
|
| morph_models
|
|
| lambda_ai_models
|
|
| inception_models
|
|
| black_forest_labs_models
|
|
| recraft_models
|
|
| cometapi_models
|
|
| oci_models
|
|
| heroku_models
|
|
| vercel_ai_gateway_models
|
|
| volcengine_models
|
|
| wandb_models
|
|
| ovhcloud_models
|
|
| lemonade_models
|
|
| docker_model_runner_models
|
|
| reducto_models
|
|
| bedrock_mantle_models
|
|
| set(clarifai_models)
|
|
)
|
|
|
|
model_list_set = set(model_list)
|
|
|
|
# provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time
|
|
|
|
|
|
def _build_models_by_provider() -> dict:
|
|
return {
|
|
"openai": open_ai_chat_completion_models | open_ai_text_completion_models,
|
|
"text-completion-openai": open_ai_text_completion_models,
|
|
"cohere": cohere_models | cohere_chat_models,
|
|
"cohere_chat": cohere_chat_models,
|
|
"anthropic": anthropic_models,
|
|
"replicate": replicate_models,
|
|
"huggingface": huggingface_models,
|
|
"together_ai": together_ai_models,
|
|
"baseten": baseten_models,
|
|
"openrouter": openrouter_models,
|
|
"vercel_ai_gateway": vercel_ai_gateway_models,
|
|
"datarobot": datarobot_models,
|
|
"vertex_ai": vertex_chat_models
|
|
| vertex_text_models
|
|
| vertex_anthropic_models
|
|
| vertex_vision_models
|
|
| vertex_language_models
|
|
| vertex_deepseek_models
|
|
| vertex_minimax_models
|
|
| vertex_moonshot_models
|
|
| vertex_zai_models,
|
|
"ai21": ai21_models,
|
|
"bedrock": bedrock_models | bedrock_converse_models,
|
|
"petals": petals_models,
|
|
"ollama": ollama_models,
|
|
"ollama_chat": ollama_models,
|
|
"deepinfra": deepinfra_models,
|
|
"perplexity": perplexity_models,
|
|
"maritalk": maritalk_models,
|
|
"watsonx": watsonx_models,
|
|
"gemini": gemini_models,
|
|
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
|
|
"aleph_alpha": aleph_alpha_models,
|
|
"text-completion-codestral": text_completion_codestral_models,
|
|
"text-completion-inception": text_completion_inception_models,
|
|
"xai": xai_models,
|
|
"zai": zai_models,
|
|
"fal_ai": fal_ai_models,
|
|
"deepseek": deepseek_models,
|
|
"tencent": tencent_models,
|
|
"runwayml": runwayml_models,
|
|
"mistral": mistral_chat_models,
|
|
"azure_ai": azure_ai_models,
|
|
"voyage": voyage_models,
|
|
"infinity": infinity_models,
|
|
"databricks": databricks_models,
|
|
"cloudflare": cloudflare_models,
|
|
"codestral": codestral_models,
|
|
"nlp_cloud": nlp_cloud_models,
|
|
"friendliai": friendliai_models,
|
|
"palm": palm_models,
|
|
"groq": groq_models,
|
|
"azure": azure_models | azure_text_models,
|
|
"azure_anthropic": azure_anthropic_models,
|
|
"azure_text": azure_text_models,
|
|
"anyscale": anyscale_models,
|
|
"cerebras": cerebras_models,
|
|
"galadriel": galadriel_models,
|
|
"nvidia_nim": nvidia_nim_models,
|
|
"nvidia_riva": nvidia_riva_models,
|
|
"soniox": soniox_models,
|
|
"sambanova": sambanova_models | sambanova_embedding_models,
|
|
"novita": novita_models,
|
|
"nebius": nebius_models | nebius_embedding_models,
|
|
"aiml": aiml_models,
|
|
"assemblyai": assemblyai_models,
|
|
"jina_ai": jina_ai_models,
|
|
"snowflake": snowflake_models,
|
|
"gradient_ai": gradient_ai_models,
|
|
"meta_llama": llama_models,
|
|
"nscale": nscale_models,
|
|
"featherless_ai": featherless_ai_models,
|
|
"deepgram": deepgram_models,
|
|
"elevenlabs": elevenlabs_models,
|
|
"heroku": heroku_models,
|
|
"dashscope": dashscope_models,
|
|
"modelscope": modelscope_models,
|
|
"moonshot": moonshot_models,
|
|
"publicai": publicai_models,
|
|
"darkbloom": darkbloom_models,
|
|
"v0": v0_models,
|
|
"morph": morph_models,
|
|
"lambda_ai": lambda_ai_models,
|
|
"inception": inception_models,
|
|
"hyperbolic": hyperbolic_models,
|
|
"black_forest_labs": black_forest_labs_models,
|
|
"recraft": recraft_models,
|
|
"cometapi": cometapi_models,
|
|
"oci": oci_models,
|
|
"volcengine": volcengine_models,
|
|
"wandb": wandb_models,
|
|
"ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
|
|
"lemonade": lemonade_models,
|
|
"clarifai": clarifai_models,
|
|
"amazon_nova": amazon_nova_models,
|
|
"stability": stability_models,
|
|
"github_copilot": github_copilot_models,
|
|
"chatgpt": chatgpt_models,
|
|
"minimax": minimax_models,
|
|
"aws_polly": aws_polly_models,
|
|
"gigachat": gigachat_models,
|
|
"llamagate": llamagate_models,
|
|
"reducto": reducto_models,
|
|
"bedrock_mantle": bedrock_mantle_models,
|
|
}
|
|
|
|
|
|
models_by_provider: dict = _build_models_by_provider()
|
|
|
|
# mapping for those models which have larger equivalents
|
|
longer_context_model_fallback_dict: dict = {
|
|
# openai chat completion models
|
|
"gpt-3.5-turbo": "gpt-3.5-turbo-16k",
|
|
"gpt-3.5-turbo-0301": "gpt-3.5-turbo-16k-0301",
|
|
"gpt-3.5-turbo-0613": "gpt-3.5-turbo-16k-0613",
|
|
"gpt-4": "gpt-4-32k",
|
|
"gpt-4-0314": "gpt-4-32k-0314",
|
|
"gpt-4-0613": "gpt-4-32k-0613",
|
|
# anthropic
|
|
"claude-instant-1": "claude-2",
|
|
"claude-instant-1.2": "claude-2",
|
|
# vertexai
|
|
"chat-bison": "chat-bison-32k",
|
|
"chat-bison@001": "chat-bison-32k",
|
|
"codechat-bison": "codechat-bison-32k",
|
|
"codechat-bison@001": "codechat-bison-32k",
|
|
# openrouter
|
|
"openrouter/openai/gpt-3.5-turbo": "openrouter/openai/gpt-3.5-turbo-16k",
|
|
"openrouter/anthropic/claude-instant-v1": "openrouter/anthropic/claude-2",
|
|
}
|
|
|
|
####### EMBEDDING MODELS ###################
|
|
|
|
all_embedding_models = (
|
|
open_ai_embedding_models
|
|
| set(cohere_embedding_models)
|
|
| set(bedrock_embedding_models)
|
|
| vertex_embedding_models
|
|
| fireworks_ai_embedding_models
|
|
| nebius_embedding_models
|
|
| sambanova_embedding_models
|
|
| ovhcloud_embedding_models
|
|
)
|
|
|
|
####### IMAGE GENERATION MODELS ###################
|
|
openai_image_generation_models = ["dall-e-2", "dall-e-3"]
|
|
|
|
####### VIDEO GENERATION MODELS ###################
|
|
openai_video_generation_models = ["sora-2"]
|
|
|
|
# timeout is lazy-loaded via __getattr__
|
|
# get_llm_provider is lazy-loaded via __getattr__
|
|
# remove_index_from_tool_calls is lazy-loaded via __getattr__
|
|
|
|
# Import KeyManagementSettings here (before utils import) because _key_management_settings
|
|
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
|
|
from litellm.types.secret_managers.main import KeyManagementSettings
|
|
|
|
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
|
|
|
|
# client must be imported immediately as it's used as a decorator at function definition time
|
|
from .utils import client
|
|
|
|
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
|
|
# (which imports tiktoken) at import time
|
|
|
|
from .llms.custom_llm import CustomLLM
|
|
from .llms.anthropic.common_utils import AnthropicModelInfo
|
|
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
|
|
from .llms.deprecated_providers.palm import (
|
|
PalmConfig,
|
|
) # here to prevent breaking changes
|
|
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
|
|
from .llms.gemini.common_utils import GeminiModelInfo
|
|
|
|
|
|
from .llms.vertex_ai.vertex_embeddings.transformation import (
|
|
VertexAITextEmbeddingConfig,
|
|
)
|
|
|
|
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
|
|
|
|
|
|
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
|
|
AmazonTitanV2Config,
|
|
)
|
|
from .llms.topaz.common_utils import TopazModelInfo
|
|
|
|
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
|
|
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
|
|
from .llms.xai.common_utils import XAIModelInfo
|
|
|
|
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
|
|
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
|
|
|
|
# Import LlmProviders here (before main import) because it's imported during import time
|
|
# in multiple places including openai.py (via main import)
|
|
from litellm.types.utils import LlmProviders
|
|
|
|
## Lazy loading this is not straightforward, will leave it here for now.
|
|
from .main import *
|
|
from .compression import compress
|
|
|
|
# Skills API
|
|
from .skills.main import (
|
|
create_skill,
|
|
acreate_skill,
|
|
list_skills,
|
|
alist_skills,
|
|
get_skill,
|
|
aget_skill,
|
|
delete_skill,
|
|
adelete_skill,
|
|
)
|
|
from .evals.main import (
|
|
create_eval,
|
|
acreate_eval,
|
|
list_evals,
|
|
alist_evals,
|
|
get_eval,
|
|
aget_eval,
|
|
delete_eval,
|
|
adelete_eval,
|
|
cancel_eval,
|
|
acancel_eval,
|
|
create_run,
|
|
acreate_run,
|
|
list_runs,
|
|
alist_runs,
|
|
get_run,
|
|
aget_run,
|
|
delete_run,
|
|
adelete_run,
|
|
cancel_run,
|
|
acancel_run,
|
|
)
|
|
from .integrations import *
|
|
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
|
from .exceptions import (
|
|
AuthenticationError,
|
|
InvalidRequestError,
|
|
BadRequestError,
|
|
ImageFetchError,
|
|
NotFoundError,
|
|
PermissionDeniedError,
|
|
RateLimitError,
|
|
RateLimitErrorCategory,
|
|
RateLimitType,
|
|
ServiceUnavailableError,
|
|
BadGatewayError,
|
|
OpenAIError,
|
|
ContextWindowExceededError,
|
|
ContentPolicyViolationError,
|
|
BudgetExceededError,
|
|
APIError,
|
|
Timeout,
|
|
APIConnectionError,
|
|
UnsupportedParamsError,
|
|
APIResponseValidationError,
|
|
UnprocessableEntityError,
|
|
InternalServerError,
|
|
JSONSchemaValidationError,
|
|
LITELLM_EXCEPTION_TYPES,
|
|
MockException,
|
|
)
|
|
from .budget_manager import BudgetManager
|
|
from .proxy.proxy_cli import run_server
|
|
from .router import Router
|
|
from .assistants.main import *
|
|
from .batches.main import *
|
|
from .images.main import *
|
|
from .videos.main import *
|
|
from .batch_completion.main import *
|
|
from .rerank_api.main import *
|
|
from .llms.anthropic.experimental_pass_through.messages.handler import *
|
|
from .responses.main import *
|
|
|
|
# Interactions API is available as litellm.interactions module
|
|
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
|
|
from . import interactions
|
|
from .interactions.agents.main import (
|
|
acreate as acreate_agent,
|
|
create as create_agent,
|
|
alist as alist_agents,
|
|
list as list_agents,
|
|
aget as aget_agent,
|
|
get as get_agent,
|
|
adelete as adelete_agent,
|
|
delete as delete_agent,
|
|
alist_versions as alist_agent_versions,
|
|
list_versions as list_agent_versions,
|
|
)
|
|
from .skills.main import (
|
|
create_skill,
|
|
acreate_skill,
|
|
list_skills,
|
|
alist_skills,
|
|
get_skill,
|
|
aget_skill,
|
|
delete_skill,
|
|
adelete_skill,
|
|
)
|
|
from .containers.main import *
|
|
from .ocr.main import *
|
|
from .rust_bridge.ocr import use_litellm_rust
|
|
from .rag.main import *
|
|
from .sandbox.main import *
|
|
from .search.main import *
|
|
from .realtime_api.main import (
|
|
_arealtime,
|
|
acreate_realtime_client_secret,
|
|
acreate_realtime_transcription_session,
|
|
arealtime_calls,
|
|
)
|
|
from .responses.main import _aresponses_websocket
|
|
from .fine_tuning.main import *
|
|
from .files.main import *
|
|
from .vector_store_files.main import (
|
|
acreate as avector_store_file_create,
|
|
adelete as avector_store_file_delete,
|
|
alist as avector_store_file_list,
|
|
aretrieve as avector_store_file_retrieve,
|
|
aretrieve_content as avector_store_file_content,
|
|
aupdate as avector_store_file_update,
|
|
create as vector_store_file_create,
|
|
delete as vector_store_file_delete,
|
|
list as vector_store_file_list,
|
|
retrieve as vector_store_file_retrieve,
|
|
retrieve_content as vector_store_file_content,
|
|
update as vector_store_file_update,
|
|
)
|
|
from .scheduler import *
|
|
|
|
### ADAPTERS ###
|
|
from .types.adapter import AdapterItem
|
|
import litellm.anthropic_interface as anthropic
|
|
|
|
adapters: List[AdapterItem] = []
|
|
|
|
### Vector Store Registry ###
|
|
from .vector_stores.vector_store_registry import (
|
|
VectorStoreRegistry,
|
|
VectorStoreIndexRegistry,
|
|
)
|
|
|
|
vector_store_registry: Optional[VectorStoreRegistry] = None
|
|
vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None
|
|
|
|
### RAG ###
|
|
from . import rag
|
|
|
|
### CUSTOM LLMs ###
|
|
from .types.llms.custom_llm import CustomLLMItem
|
|
|
|
custom_provider_map: List[CustomLLMItem] = []
|
|
_custom_providers: List[str] = [] # internal helper util, used to track names of custom providers
|
|
disable_hf_tokenizer_download: Optional[bool] = (
|
|
None # disable huggingface tokenizer download. Defaults to openai clk100
|
|
)
|
|
global_disable_no_log_param: bool = False
|
|
|
|
### CLI UTILITIES ###
|
|
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
|
|
|
|
### PASSTHROUGH ###
|
|
from .passthrough import allm_passthrough_route, llm_passthrough_route
|
|
from .google_genai import agenerate_content
|
|
|
|
### GLOBAL CONFIG ###
|
|
global_bitbucket_config: Optional[Dict[str, Any]] = None
|
|
|
|
|
|
def set_global_bitbucket_config(config: Dict[str, Any]) -> None:
|
|
"""Set global BitBucket configuration for prompt management."""
|
|
global global_bitbucket_config
|
|
global_bitbucket_config = config
|
|
|
|
|
|
### GLOBAL CONFIG ###
|
|
global_gitlab_config: Optional[Dict[str, Any]] = None
|
|
|
|
|
|
def set_global_gitlab_config(config: Dict[str, Any]) -> None:
|
|
"""Set global BitBucket configuration for prompt management."""
|
|
global global_gitlab_config
|
|
global_gitlab_config = config
|
|
|
|
|
|
# Lazy loading system for heavy modules to reduce initial import time and memory usage
|
|
|
|
if TYPE_CHECKING:
|
|
from litellm.types.utils import ModelInfo as _ModelInfoType
|
|
from litellm.types.utils import PriorityReservationSettings
|
|
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
|
from litellm.caching.caching import Cache
|
|
|
|
# Type stubs for lazy-loaded configs to help mypy
|
|
from .llms.bedrock.chat.converse_transformation import (
|
|
AmazonConverseConfig as AmazonConverseConfig,
|
|
)
|
|
from .llms.openai_like.chat.handler import (
|
|
OpenAILikeChatConfig as OpenAILikeChatConfig,
|
|
)
|
|
from .llms.galadriel.chat.transformation import (
|
|
GaladrielChatConfig as GaladrielChatConfig,
|
|
)
|
|
from .llms.github.chat.transformation import GithubChatConfig as GithubChatConfig
|
|
from .llms.azure_ai.anthropic.transformation import (
|
|
AzureAnthropicConfig as AzureAnthropicConfig,
|
|
)
|
|
from .llms.bytez.chat.transformation import BytezChatConfig as BytezChatConfig
|
|
from .llms.compactifai.chat.transformation import (
|
|
CompactifAIChatConfig as CompactifAIChatConfig,
|
|
)
|
|
from .llms.empower.chat.transformation import EmpowerChatConfig as EmpowerChatConfig
|
|
from .llms.minimax.chat.transformation import MinimaxChatConfig as MinimaxChatConfig
|
|
from .llms.aiohttp_openai.chat.transformation import (
|
|
AiohttpOpenAIChatConfig as AiohttpOpenAIChatConfig,
|
|
)
|
|
from .llms.huggingface.chat.transformation import (
|
|
HuggingFaceChatConfig as HuggingFaceChatConfig,
|
|
)
|
|
from .llms.huggingface.embedding.transformation import (
|
|
HuggingFaceEmbeddingConfig as HuggingFaceEmbeddingConfig,
|
|
)
|
|
from .llms.oobabooga.chat.transformation import OobaboogaConfig as OobaboogaConfig
|
|
from .llms.maritalk import MaritalkConfig as MaritalkConfig
|
|
from .llms.openrouter.chat.transformation import (
|
|
OpenrouterConfig as OpenrouterConfig,
|
|
)
|
|
from .llms.datarobot.chat.transformation import DataRobotConfig as DataRobotConfig
|
|
from .llms.anthropic.chat.transformation import AnthropicConfig as AnthropicConfig
|
|
from .llms.bedrock.claude_platform.transformation import (
|
|
BedrockClaudePlatformConfig as BedrockClaudePlatformConfig,
|
|
)
|
|
from .llms.bedrock.claude_platform.messages_transformation import (
|
|
BedrockClaudePlatformMessagesConfig as BedrockClaudePlatformMessagesConfig,
|
|
)
|
|
from .llms.anthropic.completion.transformation import (
|
|
AnthropicTextConfig as AnthropicTextConfig,
|
|
)
|
|
from .llms.groq.stt.transformation import GroqSTTConfig as GroqSTTConfig
|
|
from .llms.triton.completion.transformation import TritonConfig as TritonConfig
|
|
from .llms.triton.completion.transformation import (
|
|
TritonGenerateConfig as TritonGenerateConfig,
|
|
)
|
|
from .llms.triton.completion.transformation import (
|
|
TritonInferConfig as TritonInferConfig,
|
|
)
|
|
from .llms.triton.embedding.transformation import (
|
|
TritonEmbeddingConfig as TritonEmbeddingConfig,
|
|
)
|
|
from .llms.huggingface.rerank.transformation import (
|
|
HuggingFaceRerankConfig as HuggingFaceRerankConfig,
|
|
)
|
|
from .llms.databricks.chat.transformation import (
|
|
DatabricksConfig as DatabricksConfig,
|
|
)
|
|
from .llms.databricks.embed.transformation import (
|
|
DatabricksEmbeddingConfig as DatabricksEmbeddingConfig,
|
|
)
|
|
from .llms.predibase.chat.transformation import PredibaseConfig as PredibaseConfig
|
|
from .llms.replicate.chat.transformation import ReplicateConfig as ReplicateConfig
|
|
from .llms.snowflake.chat.transformation import SnowflakeConfig as SnowflakeConfig
|
|
from .llms.cohere.rerank.transformation import (
|
|
CohereRerankConfig as CohereRerankConfig,
|
|
)
|
|
from .llms.cohere.rerank_v2.transformation import (
|
|
CohereRerankV2Config as CohereRerankV2Config,
|
|
)
|
|
from .llms.azure_ai.rerank.transformation import (
|
|
AzureAIRerankConfig as AzureAIRerankConfig,
|
|
)
|
|
from .llms.infinity.rerank.transformation import (
|
|
InfinityRerankConfig as InfinityRerankConfig,
|
|
)
|
|
from .llms.jina_ai.rerank.transformation import (
|
|
JinaAIRerankConfig as JinaAIRerankConfig,
|
|
)
|
|
from .llms.deepinfra.rerank.transformation import (
|
|
DeepinfraRerankConfig as DeepinfraRerankConfig,
|
|
)
|
|
from .llms.hosted_vllm.rerank.transformation import (
|
|
HostedVLLMRerankConfig as HostedVLLMRerankConfig,
|
|
)
|
|
from .llms.nvidia_nim.rerank.transformation import (
|
|
NvidiaNimRerankConfig as NvidiaNimRerankConfig,
|
|
)
|
|
from .llms.nvidia_nim.rerank.ranking_transformation import (
|
|
NvidiaNimRankingConfig as NvidiaNimRankingConfig,
|
|
)
|
|
from .llms.vertex_ai.rerank.transformation import (
|
|
VertexAIRerankConfig as VertexAIRerankConfig,
|
|
)
|
|
from .llms.fireworks_ai.rerank.transformation import (
|
|
FireworksAIRerankConfig as FireworksAIRerankConfig,
|
|
)
|
|
from .llms.voyage.rerank.transformation import (
|
|
VoyageRerankConfig as VoyageRerankConfig,
|
|
)
|
|
from .llms.watsonx.rerank.transformation import (
|
|
IBMWatsonXRerankConfig as IBMWatsonXRerankConfig,
|
|
)
|
|
from .llms.clarifai.chat.transformation import ClarifaiConfig as ClarifaiConfig
|
|
from .llms.ai21.chat.transformation import AI21ChatConfig as AI21ChatConfig
|
|
from .llms.meta_llama.chat.transformation import LlamaAPIConfig as LlamaAPIConfig
|
|
from .llms.together_ai.completion.transformation import (
|
|
TogetherAITextCompletionConfig as TogetherAITextCompletionConfig,
|
|
)
|
|
from .llms.cloudflare.chat.transformation import (
|
|
CloudflareChatConfig as CloudflareChatConfig,
|
|
)
|
|
from .llms.novita.chat.transformation import NovitaConfig as NovitaConfig
|
|
from .llms.petals.completion.transformation import PetalsConfig as PetalsConfig
|
|
from .llms.ollama.chat.transformation import OllamaChatConfig as OllamaChatConfig
|
|
from .llms.ollama.completion.transformation import OllamaConfig as OllamaConfig
|
|
from .llms.sagemaker.completion.transformation import (
|
|
SagemakerConfig as SagemakerConfig,
|
|
)
|
|
from .llms.sagemaker.chat.transformation import (
|
|
SagemakerChatConfig as SagemakerChatConfig,
|
|
)
|
|
from .llms.sagemaker.nova.transformation import (
|
|
SagemakerNovaConfig as SagemakerNovaConfig,
|
|
)
|
|
from .llms.cohere.chat.transformation import CohereChatConfig as CohereChatConfig
|
|
from .llms.anthropic.experimental_pass_through.messages.transformation import (
|
|
AnthropicMessagesConfig as AnthropicMessagesConfig,
|
|
)
|
|
from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig,
|
|
)
|
|
from .llms.bedrock.messages.mantle_transformation import (
|
|
AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
|
|
)
|
|
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
|
|
from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
|
|
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
|
VertexGeminiConfig as VertexGeminiConfig,
|
|
)
|
|
from .llms.gemini.chat.transformation import (
|
|
GoogleAIStudioGeminiConfig as GoogleAIStudioGeminiConfig,
|
|
)
|
|
from .llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import (
|
|
VertexAIAnthropicConfig as VertexAIAnthropicConfig,
|
|
)
|
|
from .llms.vertex_ai.vertex_ai_partner_models.llama3.transformation import (
|
|
VertexAILlama3Config as VertexAILlama3Config,
|
|
)
|
|
from .llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import (
|
|
VertexAIAi21Config as VertexAIAi21Config,
|
|
)
|
|
from .llms.bedrock.chat.invoke_handler import (
|
|
AmazonCohereChatConfig as AmazonCohereChatConfig,
|
|
)
|
|
from .llms.bedrock.common_utils import (
|
|
AmazonBedrockGlobalConfig as AmazonBedrockGlobalConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_ai21_transformation import (
|
|
AmazonAI21Config as AmazonAI21Config,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_nova_transformation import (
|
|
AmazonInvokeNovaConfig as AmazonInvokeNovaConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_qwen2_transformation import (
|
|
AmazonQwen2Config as AmazonQwen2Config,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import (
|
|
AmazonQwen3Config as AmazonQwen3Config,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.anthropic_claude2_transformation import (
|
|
AmazonAnthropicConfig as AmazonAnthropicConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
|
|
AmazonAnthropicClaudeConfig as AmazonAnthropicClaudeConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_cohere_transformation import (
|
|
AmazonCohereConfig as AmazonCohereConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_llama_transformation import (
|
|
AmazonLlamaConfig as AmazonLlamaConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_deepseek_transformation import (
|
|
AmazonDeepSeekR1Config as AmazonDeepSeekR1Config,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation import (
|
|
AmazonMistralConfig as AmazonMistralConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_moonshot_transformation import (
|
|
AmazonMoonshotConfig as AmazonMoonshotConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import (
|
|
AmazonTitanConfig as AmazonTitanConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import (
|
|
AmazonTwelveLabsPegasusConfig as AmazonTwelveLabsPegasusConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
|
|
AmazonInvokeConfig as AmazonInvokeConfig,
|
|
)
|
|
from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
|
|
AmazonBedrockOpenAIConfig as AmazonBedrockOpenAIConfig,
|
|
)
|
|
from .llms.bedrock.image_generation.amazon_stability1_transformation import (
|
|
AmazonStabilityConfig as AmazonStabilityConfig,
|
|
)
|
|
from .llms.bedrock.image_generation.amazon_stability3_transformation import (
|
|
AmazonStability3Config as AmazonStability3Config,
|
|
)
|
|
from .llms.bedrock.image_generation.amazon_nova_canvas_transformation import (
|
|
AmazonNovaCanvasConfig as AmazonNovaCanvasConfig,
|
|
)
|
|
from .llms.bedrock.embed.amazon_titan_g1_transformation import (
|
|
AmazonTitanG1Config as AmazonTitanG1Config,
|
|
)
|
|
from .llms.bedrock.embed.amazon_titan_multimodal_transformation import (
|
|
AmazonTitanMultimodalEmbeddingG1Config as AmazonTitanMultimodalEmbeddingG1Config,
|
|
)
|
|
from .llms.cohere.chat.v2_transformation import (
|
|
CohereV2ChatConfig as CohereV2ChatConfig,
|
|
)
|
|
from .llms.bedrock.embed.cohere_transformation import (
|
|
BedrockCohereEmbeddingConfig as BedrockCohereEmbeddingConfig,
|
|
)
|
|
from .llms.bedrock.embed.twelvelabs_marengo_transformation import (
|
|
TwelveLabsMarengoEmbeddingConfig as TwelveLabsMarengoEmbeddingConfig,
|
|
)
|
|
from .llms.bedrock.embed.amazon_nova_transformation import (
|
|
AmazonNovaEmbeddingConfig as AmazonNovaEmbeddingConfig,
|
|
)
|
|
from .llms.openai.openai import (
|
|
OpenAIConfig as OpenAIConfig,
|
|
MistralEmbeddingConfig as MistralEmbeddingConfig,
|
|
)
|
|
from .llms.openai.image_variations.transformation import (
|
|
OpenAIImageVariationConfig as OpenAIImageVariationConfig,
|
|
)
|
|
from .llms.deepgram.audio_transcription.transformation import (
|
|
DeepgramAudioTranscriptionConfig as DeepgramAudioTranscriptionConfig,
|
|
)
|
|
from .llms.nvidia_riva.audio_transcription.transformation import (
|
|
NvidiaRivaAudioTranscriptionConfig as NvidiaRivaAudioTranscriptionConfig,
|
|
)
|
|
from .llms.topaz.image_variations.transformation import (
|
|
TopazImageVariationConfig as TopazImageVariationConfig,
|
|
)
|
|
from litellm.llms.openai.completion.transformation import (
|
|
OpenAITextCompletionConfig as OpenAITextCompletionConfig,
|
|
)
|
|
from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig
|
|
from .llms.bedrock_mantle.chat.transformation import (
|
|
BedrockMantleChatConfig as BedrockMantleChatConfig,
|
|
)
|
|
from .llms.a2a.chat.transformation import A2AConfig as A2AConfig
|
|
from .llms.voyage.embedding.transformation import (
|
|
VoyageEmbeddingConfig as VoyageEmbeddingConfig,
|
|
)
|
|
from .llms.voyage.embedding.transformation_contextual import (
|
|
VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig,
|
|
)
|
|
from .llms.voyage.embedding.transformation_multimodal import (
|
|
VoyageMultimodalEmbeddingConfig as VoyageMultimodalEmbeddingConfig,
|
|
)
|
|
from .llms.infinity.embedding.transformation import (
|
|
InfinityEmbeddingConfig as InfinityEmbeddingConfig,
|
|
)
|
|
from .llms.perplexity.embedding.transformation import (
|
|
PerplexityEmbeddingConfig as PerplexityEmbeddingConfig,
|
|
)
|
|
from .llms.azure_ai.chat.transformation import (
|
|
AzureAIStudioConfig as AzureAIStudioConfig,
|
|
)
|
|
from .llms.mistral.chat.transformation import MistralConfig as MistralConfig
|
|
from .llms.openai.responses.transformation import (
|
|
OpenAIResponsesAPIConfig as OpenAIResponsesAPIConfig,
|
|
)
|
|
from .llms.azure.responses.transformation import (
|
|
AzureOpenAIResponsesAPIConfig as AzureOpenAIResponsesAPIConfig,
|
|
)
|
|
from .llms.azure.responses.o_series_transformation import (
|
|
AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig,
|
|
)
|
|
from .llms.xai.responses.transformation import (
|
|
XAIResponsesAPIConfig as XAIResponsesAPIConfig,
|
|
)
|
|
from .llms.litellm_proxy.responses.transformation import (
|
|
LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig,
|
|
)
|
|
from .llms.volcengine.responses.transformation import (
|
|
VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig,
|
|
)
|
|
from .llms.manus.responses.transformation import (
|
|
ManusResponsesAPIConfig as ManusResponsesAPIConfig,
|
|
)
|
|
from .llms.perplexity.responses.transformation import (
|
|
PerplexityResponsesConfig as PerplexityResponsesConfig,
|
|
)
|
|
from .llms.databricks.responses.transformation import (
|
|
DatabricksResponsesAPIConfig as DatabricksResponsesAPIConfig,
|
|
)
|
|
from .llms.openrouter.responses.transformation import (
|
|
OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig,
|
|
)
|
|
from .llms.bedrock_mantle.responses.transformation import (
|
|
BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig,
|
|
)
|
|
from .llms.gemini.interactions.transformation import (
|
|
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
|
|
)
|
|
from .llms.openai.chat.o_series_transformation import (
|
|
OpenAIOSeriesConfig as OpenAIOSeriesConfig,
|
|
OpenAIOSeriesConfig as OpenAIO1Config,
|
|
)
|
|
from .llms.anthropic.skills.transformation import (
|
|
AnthropicSkillsConfig as AnthropicSkillsConfig,
|
|
)
|
|
from .llms.base_llm.skills.transformation import (
|
|
BaseSkillsAPIConfig as BaseSkillsAPIConfig,
|
|
)
|
|
from .llms.gradient_ai.chat.transformation import (
|
|
GradientAIConfig as GradientAIConfig,
|
|
)
|
|
from .llms.openai.chat.gpt_transformation import OpenAIGPTConfig as OpenAIGPTConfig
|
|
from .llms.openai.chat.gpt_5_transformation import (
|
|
OpenAIGPT5Config as OpenAIGPT5Config,
|
|
)
|
|
from .llms.openai.transcriptions.whisper_transformation import (
|
|
OpenAIWhisperAudioTranscriptionConfig as OpenAIWhisperAudioTranscriptionConfig,
|
|
)
|
|
from .llms.openai.transcriptions.gpt_transformation import (
|
|
OpenAIGPTAudioTranscriptionConfig as OpenAIGPTAudioTranscriptionConfig,
|
|
)
|
|
from .llms.openai.chat.gpt_audio_transformation import (
|
|
OpenAIGPTAudioConfig as OpenAIGPTAudioConfig,
|
|
)
|
|
from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig
|
|
from .llms.nvidia_nim.embed import (
|
|
NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig,
|
|
)
|
|
from .llms.gdc.chat.transformation import GDCGeminiConfig as GDCGeminiConfig
|
|
|
|
# Type stubs for lazy-loaded config instances
|
|
openaiOSeriesConfig: OpenAIOSeriesConfig
|
|
openAIGPTConfig: OpenAIGPTConfig
|
|
openAIGPTAudioConfig: OpenAIGPTAudioConfig
|
|
openAIGPT5Config: OpenAIGPT5Config
|
|
nvidiaNimConfig: NvidiaNimConfig
|
|
nvidiaNimEmbeddingConfig: NvidiaNimEmbeddingConfig
|
|
|
|
# Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference
|
|
from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig
|
|
from .llms.deepseek.chat.transformation import (
|
|
DeepSeekChatConfig as _DeepSeekChatConfig,
|
|
)
|
|
from .llms.tencent.chat.transformation import (
|
|
TencentChatConfig as _TencentChatConfig,
|
|
)
|
|
from .llms.sap.chat.transformation import (
|
|
GenAIHubOrchestrationConfig as _GenAIHubOrchestrationConfig,
|
|
)
|
|
from .llms.sap.embed.transformation import (
|
|
GenAIHubEmbeddingConfig as _GenAIHubEmbeddingConfig,
|
|
)
|
|
from .llms.azure.chat.o_series_transformation import (
|
|
AzureOpenAIO1Config as _AzureOpenAIO1Config,
|
|
)
|
|
from .llms.perplexity.chat.transformation import (
|
|
PerplexityChatConfig as _PerplexityChatConfig,
|
|
)
|
|
from .llms.nscale.chat.transformation import NscaleConfig as _NscaleConfig
|
|
from .llms.watsonx.chat.transformation import (
|
|
IBMWatsonXChatConfig as _IBMWatsonXChatConfig,
|
|
)
|
|
from .llms.watsonx.completion.transformation import (
|
|
IBMWatsonXAIConfig as _IBMWatsonXAIConfig,
|
|
)
|
|
from .llms.litellm_proxy.chat.transformation import (
|
|
LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
|
|
)
|
|
from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
|
|
from .llms.llamafile.chat.transformation import (
|
|
LlamafileChatConfig as _LlamafileChatConfig,
|
|
)
|
|
from .llms.lm_studio.chat.transformation import (
|
|
LMStudioChatConfig as _LMStudioChatConfig,
|
|
)
|
|
from .llms.lm_studio.embed.transformation import (
|
|
LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig,
|
|
)
|
|
from .llms.watsonx.embed.transformation import (
|
|
IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig,
|
|
)
|
|
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
|
VertexGeminiConfig as _VertexGeminiConfig,
|
|
)
|
|
|
|
# Type stubs for lazy-loaded config classes (to help mypy understand types)
|
|
VLLMConfig: Type[_VLLMConfig]
|
|
DeepSeekChatConfig: Type[_DeepSeekChatConfig]
|
|
TencentChatConfig: Type[_TencentChatConfig]
|
|
GenAIHubOrchestrationConfig: Type[_GenAIHubOrchestrationConfig]
|
|
GenAIHubEmbeddingConfig: Type[_GenAIHubEmbeddingConfig]
|
|
AzureOpenAIO1Config: Type[_AzureOpenAIO1Config]
|
|
PerplexityChatConfig: Type[_PerplexityChatConfig]
|
|
NscaleConfig: Type[_NscaleConfig]
|
|
IBMWatsonXChatConfig: Type[_IBMWatsonXChatConfig]
|
|
IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
|
|
LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
|
|
DeepInfraConfig: Type[_DeepInfraConfig]
|
|
LlamafileChatConfig: Type[_LlamafileChatConfig]
|
|
LMStudioChatConfig: Type[_LMStudioChatConfig]
|
|
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
|
|
IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig]
|
|
VertexAIConfig: Type[_VertexGeminiConfig] # Alias for VertexGeminiConfig
|
|
|
|
from .llms.featherless_ai.chat.transformation import (
|
|
FeatherlessAIConfig as FeatherlessAIConfig,
|
|
)
|
|
from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig
|
|
from .llms.baseten.chat import BasetenConfig as BasetenConfig
|
|
from .llms.sambanova.chat import SambanovaConfig as SambanovaConfig
|
|
from .llms.sambanova.embedding.transformation import (
|
|
SambaNovaEmbeddingConfig as SambaNovaEmbeddingConfig,
|
|
)
|
|
from .llms.fireworks_ai.chat.transformation import (
|
|
FireworksAIConfig as FireworksAIConfig,
|
|
)
|
|
from .llms.fireworks_ai.completion.transformation import (
|
|
FireworksAITextCompletionConfig as FireworksAITextCompletionConfig,
|
|
)
|
|
from .llms.fireworks_ai.embed.fireworks_ai_transformation import (
|
|
FireworksAIEmbeddingConfig as FireworksAIEmbeddingConfig,
|
|
)
|
|
from .llms.friendliai.chat.transformation import (
|
|
FriendliaiChatConfig as FriendliaiChatConfig,
|
|
)
|
|
from .llms.jina_ai.embedding.transformation import (
|
|
JinaAIEmbeddingConfig as JinaAIEmbeddingConfig,
|
|
)
|
|
from .llms.xai.chat.transformation import XAIChatConfig as XAIChatConfig
|
|
from .llms.zai.chat.transformation import ZAIChatConfig as ZAIChatConfig
|
|
from .llms.aiml.chat.transformation import AIMLChatConfig as AIMLChatConfig
|
|
from .llms.volcengine.chat.transformation import (
|
|
VolcEngineChatConfig as VolcEngineChatConfig,
|
|
VolcEngineChatConfig as VolcEngineConfig,
|
|
)
|
|
from .llms.codestral.completion.transformation import (
|
|
CodestralTextCompletionConfig as CodestralTextCompletionConfig,
|
|
)
|
|
from .llms.inception.completion.transformation import (
|
|
InceptionTextCompletionConfig as InceptionTextCompletionConfig,
|
|
)
|
|
from .llms.azure.azure import (
|
|
AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig,
|
|
)
|
|
from .llms.heroku.chat.transformation import HerokuChatConfig as HerokuChatConfig
|
|
from .llms.cometapi.chat.transformation import CometAPIConfig as CometAPIConfig
|
|
from .llms.azure.chat.gpt_transformation import (
|
|
AzureOpenAIConfig as AzureOpenAIConfig,
|
|
)
|
|
from .llms.azure.chat.gpt_5_transformation import (
|
|
AzureOpenAIGPT5Config as AzureOpenAIGPT5Config,
|
|
)
|
|
from .llms.azure.completion.transformation import (
|
|
AzureOpenAITextConfig as AzureOpenAITextConfig,
|
|
)
|
|
from .llms.azure.audio_transcription.transformation import (
|
|
AzureSpeechAudioTranscriptionConfig as AzureSpeechAudioTranscriptionConfig,
|
|
)
|
|
from .llms.hosted_vllm.chat.transformation import (
|
|
HostedVLLMChatConfig as HostedVLLMChatConfig,
|
|
)
|
|
from .llms.hosted_vllm.embedding.transformation import (
|
|
HostedVLLMEmbeddingConfig as HostedVLLMEmbeddingConfig,
|
|
)
|
|
from .llms.hosted_vllm.responses.transformation import (
|
|
HostedVLLMResponsesAPIConfig as HostedVLLMResponsesAPIConfig,
|
|
)
|
|
from .llms.github_copilot.chat.transformation import (
|
|
GithubCopilotConfig as GithubCopilotConfig,
|
|
)
|
|
from .llms.github_copilot.responses.transformation import (
|
|
GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig,
|
|
)
|
|
from .llms.github_copilot.embedding.transformation import (
|
|
GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig,
|
|
)
|
|
from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig
|
|
from .llms.chatgpt.responses.transformation import (
|
|
ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig,
|
|
)
|
|
from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig
|
|
from .llms.gigachat.embedding.transformation import (
|
|
GigaChatEmbeddingConfig as GigaChatEmbeddingConfig,
|
|
)
|
|
from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig
|
|
from .llms.wandb.chat.transformation import WandbConfig as WandbConfig
|
|
from .llms.dashscope.chat.transformation import (
|
|
DashScopeChatConfig as DashScopeChatConfig,
|
|
)
|
|
from .llms.dashscope.embed.transformation import (
|
|
DashScopeEmbeddingConfig as DashScopeEmbeddingConfig,
|
|
)
|
|
from .llms.dashscope.rerank.transformation import (
|
|
DashScopeRerankConfig as DashScopeRerankConfig,
|
|
)
|
|
from .llms.modelscope.chat.transformation import (
|
|
ModelScopeChatConfig as ModelScopeChatConfig,
|
|
)
|
|
from .llms.moonshot.chat.transformation import (
|
|
MoonshotChatConfig as MoonshotChatConfig,
|
|
)
|
|
from .llms.docker_model_runner.chat.transformation import (
|
|
DockerModelRunnerChatConfig as DockerModelRunnerChatConfig,
|
|
)
|
|
from .llms.v0.chat.transformation import V0ChatConfig as V0ChatConfig
|
|
from .llms.oci.chat.transformation import OCIChatConfig as OCIChatConfig
|
|
from .llms.oci.embed.transformation import OCIEmbeddingConfig as OCIEmbeddingConfig
|
|
from .llms.morph.chat.transformation import MorphChatConfig as MorphChatConfig
|
|
from .llms.ragflow.chat.transformation import RAGFlowConfig as RAGFlowConfig
|
|
from .llms.lambda_ai.chat.transformation import (
|
|
LambdaAIChatConfig as LambdaAIChatConfig,
|
|
)
|
|
from .llms.inception.chat.transformation import (
|
|
InceptionChatConfig as InceptionChatConfig,
|
|
)
|
|
from .llms.hyperbolic.chat.transformation import (
|
|
HyperbolicChatConfig as HyperbolicChatConfig,
|
|
)
|
|
from .llms.vercel_ai_gateway.chat.transformation import (
|
|
VercelAIGatewayConfig as VercelAIGatewayConfig,
|
|
)
|
|
from .llms.ovhcloud.chat.transformation import (
|
|
OVHCloudChatConfig as OVHCloudChatConfig,
|
|
)
|
|
from .llms.ovhcloud.embedding.transformation import (
|
|
OVHCloudEmbeddingConfig as OVHCloudEmbeddingConfig,
|
|
)
|
|
from .llms.cometapi.embed.transformation import (
|
|
CometAPIEmbeddingConfig as CometAPIEmbeddingConfig,
|
|
)
|
|
from .llms.lemonade.chat.transformation import (
|
|
LemonadeChatConfig as LemonadeChatConfig,
|
|
)
|
|
from .llms.snowflake.embedding.transformation import (
|
|
SnowflakeEmbeddingConfig as SnowflakeEmbeddingConfig,
|
|
)
|
|
from .llms.amazon_nova.chat.transformation import (
|
|
AmazonNovaChatConfig as AmazonNovaChatConfig,
|
|
)
|
|
from litellm.caching.llm_caching_handler import LLMClientCache
|
|
from litellm.types.llms.bedrock import COHERE_EMBEDDING_INPUT_TYPES
|
|
from litellm.types.utils import (
|
|
BudgetConfig,
|
|
CredentialItem,
|
|
PriorityReservationDict,
|
|
StandardKeyGenerationConfig,
|
|
)
|
|
from litellm.types.guardrails import GuardrailItem
|
|
from litellm.types.proxy.management_endpoints.ui_sso import (
|
|
DefaultTeamSSOParams,
|
|
LiteLLM_UpperboundKeyGenerateParams,
|
|
)
|
|
|
|
# Cost calculator functions
|
|
cost_per_token: Callable[..., Tuple[float, float]]
|
|
completion_cost: Callable[..., float]
|
|
response_cost_calculator: Any
|
|
modify_integration: Any
|
|
|
|
# Utils functions - type stubs for truly lazy loaded functions only
|
|
# (functions NOT imported via "from .main import *")
|
|
get_response_string: Callable[..., str]
|
|
supports_function_calling: Callable[..., bool]
|
|
supports_web_search: Callable[..., bool]
|
|
supports_url_context: Callable[..., bool]
|
|
supports_response_schema: Callable[..., bool]
|
|
supports_parallel_function_calling: Callable[..., bool]
|
|
supports_vision: Callable[..., bool]
|
|
supports_audio_input: Callable[..., bool]
|
|
supports_audio_output: Callable[..., bool]
|
|
supports_system_messages: Callable[..., bool]
|
|
supports_reasoning: Callable[..., bool]
|
|
acreate: Callable[..., Any]
|
|
get_max_tokens: Callable[..., int]
|
|
get_model_info: Callable[..., _ModelInfoType]
|
|
register_prompt_template: Callable[..., None]
|
|
validate_environment: Callable[..., dict]
|
|
check_valid_key: Callable[..., bool]
|
|
register_model: Callable[..., None]
|
|
encode: Callable[..., list]
|
|
decode: Callable[..., str]
|
|
_calculate_retry_after: Callable[..., float]
|
|
_should_retry: Callable[..., bool]
|
|
get_supported_openai_params: Callable[..., Optional[list]]
|
|
get_api_base: Callable[..., Optional[str]]
|
|
get_first_chars_messages: Callable[..., str]
|
|
get_provider_fields: Callable[..., List]
|
|
get_valid_models: Callable[..., list]
|
|
remove_index_from_tool_calls: Callable[..., None]
|
|
|
|
# Response types - truly lazy loaded only (not in main.py or elsewhere)
|
|
ModelResponseListIterator: Type[Any]
|
|
|
|
# HTTP handler singletons (created lazily via __getattr__ at runtime)
|
|
module_level_aclient: AsyncHTTPHandler
|
|
module_level_client: HTTPHandler
|
|
|
|
# Bedrock tool name mappings instance (lazy-loaded)
|
|
from litellm.caching.caching import InMemoryCache
|
|
|
|
bedrock_tool_name_mappings: InMemoryCache
|
|
|
|
# Azure exception class (lazy-loaded)
|
|
from litellm.llms.azure.common_utils import AzureOpenAIError
|
|
|
|
# Secret manager types (lazy-loaded)
|
|
from litellm.types.secret_managers.main import (
|
|
KeyManagementSystem,
|
|
KeyManagementSettings, # Not lazy-loaded - needed for _key_management_settings initialization
|
|
)
|
|
|
|
# Custom logger class (lazy-loaded)
|
|
from litellm.integrations.custom_logger import CustomLogger
|
|
|
|
# Datadog LLM observability params (lazy-loaded)
|
|
from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams
|
|
|
|
# Logging callback manager class and instance (lazy-loaded)
|
|
from litellm.litellm_core_utils.logging_callback_manager import (
|
|
LoggingCallbackManager,
|
|
)
|
|
|
|
logging_callback_manager: LoggingCallbackManager
|
|
|
|
# provider_list is lazy-loaded
|
|
from litellm.types.utils import LlmProviders
|
|
|
|
provider_list: List[Union[LlmProviders, str]]
|
|
|
|
# Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block
|
|
|
|
|
|
# Track if async client cleanup has been registered (for lazy loading)
|
|
_async_client_cleanup_registered = False
|
|
|
|
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
|
|
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
|
|
# For now, this only affects encoding (tiktoken) as it was the only reported issue
|
|
# See: https://github.com/BerriAI/litellm/issues/18659
|
|
# This ensures encoding is initialized before VCR starts recording HTTP requests
|
|
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
|
|
# Load encoding at import time (pre-#18070 behavior)
|
|
# This ensures encoding is initialized before VCR starts recording
|
|
from .main import encoding
|
|
|
|
|
|
def __getattr__(name: str) -> Any:
|
|
"""Lazy import handler with cached registry for improved performance."""
|
|
global _async_client_cleanup_registered
|
|
# Register async client cleanup on first access (only once)
|
|
if not _async_client_cleanup_registered:
|
|
from litellm.llms.custom_httpx.async_client_cleanup import (
|
|
register_async_client_cleanup,
|
|
)
|
|
|
|
register_async_client_cleanup()
|
|
_async_client_cleanup_registered = True
|
|
|
|
# Use cached registry from _lazy_imports instead of importing tuples every time
|
|
from ._lazy_imports import _get_lazy_import_registry
|
|
|
|
registry: Final = _get_lazy_import_registry()
|
|
|
|
# Check if name is in registry and call the cached handler function
|
|
if name in registry:
|
|
handler_func: Final = registry[name]
|
|
return handler_func(name)
|
|
|
|
# Lazy load encoding from main.py to avoid heavy tiktoken import
|
|
if name == "encoding":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
# Check if already cached
|
|
if "encoding" not in _globals:
|
|
from .main import encoding as _encoding
|
|
|
|
_globals["encoding"] = _encoding
|
|
return _globals["encoding"]
|
|
|
|
# Lazy load bedrock_tool_name_mappings instance
|
|
if name == "bedrock_tool_name_mappings":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
# Check if already cached
|
|
if "bedrock_tool_name_mappings" not in _globals:
|
|
from .llms.bedrock.chat.invoke_handler import (
|
|
bedrock_tool_name_mappings as _bedrock_tool_name_mappings,
|
|
)
|
|
|
|
_globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings
|
|
return _globals["bedrock_tool_name_mappings"]
|
|
|
|
# Lazy load AzureOpenAIError exception class
|
|
if name == "AzureOpenAIError":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
# Check if already cached
|
|
if "AzureOpenAIError" not in _globals:
|
|
from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError
|
|
|
|
_globals["AzureOpenAIError"] = _AzureOpenAIError
|
|
return _globals["AzureOpenAIError"]
|
|
|
|
# Lazy load openaiOSeriesConfig instance
|
|
if name == "openaiOSeriesConfig":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
if "openaiOSeriesConfig" not in _globals:
|
|
# Import the config class and instantiate it
|
|
config_class = __getattr__("OpenAIOSeriesConfig")
|
|
_globals["openaiOSeriesConfig"] = config_class()
|
|
return _globals["openaiOSeriesConfig"]
|
|
|
|
# Lazy load other config instances
|
|
_config_instances: Final = {
|
|
"openAIGPTConfig": "OpenAIGPTConfig",
|
|
"openAIGPTAudioConfig": "OpenAIGPTAudioConfig",
|
|
"openAIGPT5Config": "OpenAIGPT5Config",
|
|
"nvidiaNimConfig": "NvidiaNimConfig",
|
|
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
|
|
}
|
|
if name in _config_instances:
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
if name not in _globals:
|
|
# Import the config class and instantiate it
|
|
config_class = __getattr__(_config_instances[name])
|
|
_globals[name] = config_class()
|
|
return _globals[name]
|
|
|
|
# Handle OpenAIO1Config alias
|
|
if name == "OpenAIO1Config":
|
|
return __getattr__("OpenAIOSeriesConfig")
|
|
|
|
# Lazy load provider_list
|
|
if name == "provider_list":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
# Check if already cached
|
|
if "provider_list" not in _globals:
|
|
# LlmProviders is eagerly imported above, so we can import it directly
|
|
from litellm.types.utils import LlmProviders
|
|
|
|
_globals["provider_list"] = list(LlmProviders)
|
|
return _globals["provider_list"]
|
|
|
|
# Lazy load priority_reservation_settings instance
|
|
if name == "priority_reservation_settings":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
# Check if already cached
|
|
if "priority_reservation_settings" not in _globals:
|
|
# Import the class and instantiate it
|
|
PriorityReservationSettings: Final = __getattr__("PriorityReservationSettings")
|
|
_globals["priority_reservation_settings"] = PriorityReservationSettings()
|
|
return _globals["priority_reservation_settings"]
|
|
|
|
# Lazy load logging_callback_manager instance
|
|
if name == "logging_callback_manager":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
# Check if already cached
|
|
if "logging_callback_manager" not in _globals:
|
|
# Import the class and instantiate it
|
|
LoggingCallbackManager: Final = __getattr__("LoggingCallbackManager")
|
|
_globals["logging_callback_manager"] = LoggingCallbackManager()
|
|
return _globals["logging_callback_manager"]
|
|
|
|
# Lazy load _service_logger module
|
|
if name == "_service_logger":
|
|
from ._lazy_imports import get_litellm_globals
|
|
|
|
_globals = get_litellm_globals()
|
|
# Check if already cached
|
|
if "_service_logger" not in _globals:
|
|
# Import the module lazily
|
|
import litellm._service_logger
|
|
|
|
_globals["_service_logger"] = litellm._service_logger
|
|
return _globals["_service_logger"]
|
|
|
|
# Lazy load evals module functions
|
|
if name in [
|
|
"acreate_eval",
|
|
"alist_evals",
|
|
"aget_eval",
|
|
"aupdate_eval",
|
|
"adelete_eval",
|
|
"acancel_eval",
|
|
"create_eval",
|
|
"list_evals",
|
|
"get_eval",
|
|
"update_eval",
|
|
"delete_eval",
|
|
"cancel_eval",
|
|
"acreate_run",
|
|
"alist_runs",
|
|
"aget_run",
|
|
"acancel_run",
|
|
"adelete_run",
|
|
"create_run",
|
|
"list_runs",
|
|
"get_run",
|
|
"cancel_run",
|
|
"delete_run",
|
|
]:
|
|
from litellm.evals.main import (
|
|
acreate_eval,
|
|
alist_evals,
|
|
aget_eval,
|
|
aupdate_eval,
|
|
adelete_eval,
|
|
acancel_eval,
|
|
create_eval,
|
|
list_evals,
|
|
get_eval,
|
|
update_eval,
|
|
delete_eval,
|
|
cancel_eval,
|
|
acreate_run,
|
|
alist_runs,
|
|
aget_run,
|
|
acancel_run,
|
|
adelete_run,
|
|
create_run,
|
|
list_runs,
|
|
get_run,
|
|
cancel_run,
|
|
delete_run,
|
|
)
|
|
|
|
return locals()[name]
|
|
|
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
|
|
|
|
# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time
|