Merge remote-tracking branch 'origin/main' into litellm_per_second_pricing_responses_messages

# Conflicts:
#	litellm/cost_calculator.py
#	tests/test_litellm/test_cost_calculator.py
This commit is contained in:
mateo-berri 2026-09-21 22:31:30 -07:00
commit e411bc4444
306 changed files with 13362 additions and 3671 deletions

View file

@ -15,6 +15,12 @@ description: >-
cache the same directory for different workloads, and a shared key would let
whichever ran first deny the others a save.
inputs:
profile:
description: "Cargo profile the build uses (dev or release)"
required: false
default: "dev"
runs:
using: composite
steps:
@ -25,6 +31,6 @@ runs:
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-maturin-dev-
${{ runner.os }}-maturin-${{ inputs.profile }}-

View file

@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile(
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
)
HARNESS: Final = re.compile(
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"

View file

@ -134,7 +134,16 @@ def main(
uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
native_path.parent.mkdir(parents=True, exist_ok=True)
native_path.write_bytes(archive.read(native_member))
native_bytes: Final = archive.read(native_member)
native_path.write_bytes(native_bytes)
duplicated_vocabularies: Final = tuple(
member.filename
for member in wheel_members
if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
and member.file_size > 0
and archive.read(member) in native_bytes
)
wheel_metadata_tags_match: Final = (
len(wheel_metadata_tags) == len(expanded_filename_tags)
@ -205,7 +214,7 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
native_size_limit: Final = 40_000_000
native_size_limit: Final = 35_000_000
native_size_within_limit: Final = native_member.file_size <= native_size_limit
validations: Final = (
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@ -223,6 +232,7 @@ def main(
("Native module loads", native_module_loads),
("Production module omits the panic test hook", panic_test_hook_absent),
(f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
("Wheel contents are valid", not unexpected_members),
)

View file

@ -13,6 +13,7 @@ on:
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
- ".github/scripts/uv_sync_with_retries.sh"
pull_request:
branches:
- main
@ -25,6 +26,7 @@ on:
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
- ".github/scripts/uv_sync_with_retries.sh"
# Allow CodSpeed to trigger backtest performance analysis
# in order to generate initial data
workflow_dispatch:
@ -59,19 +61,27 @@ jobs:
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
with:
profile: release
# Build the wheel and resolve every dependency outside the CodSpeed
# runner: the same maturin build took 42 minutes inside `codspeed run`
# versus under 3 minutes as a plain step (LIT-6183)
- name: Build environment
- name: Build the release wheel
run: uv build --wheel --out-dir dist
- name: Install the wheel into the benchmark environment
run: |
UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
- name: Collect benchmarks
env:
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
run: >
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
uv run --frozen --no-default-groups
--with pytest==8.3.5
--with pytest-codspeed==4.3.0
--with "mcp>=2.2.0,<3.0"
--with "a2a-sdk>=1.1.0,<2.0"
pytest
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
--import-mode=importlib
-p pytest_codspeed.plugin
tests/benchmarks/
--codspeed
@ -82,13 +92,9 @@ jobs:
with:
mode: simulation
run: >
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
uv run --frozen --no-default-groups
--with pytest==8.3.5
--with pytest-codspeed==4.3.0
--with "mcp>=2.2.0,<3.0"
--with "a2a-sdk>=1.1.0,<2.0"
pytest
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
--import-mode=importlib
-p pytest_codspeed.plugin
tests/benchmarks/
--codspeed

View file

@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma

View file

@ -1,8 +1,11 @@
"""Guard the cost map on pull requests.
Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
files and may only add or update models.
"""
from __future__ import annotations
@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
)
def touches_cost_map(changed_files: Sequence[str]) -> bool:
return any(path in GUARDED_PATHS for path in changed_files)
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
if bot:
return "bot contract enforced"
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
if not bot and not touches_cost_map(changed_files):
return ()
head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
if isinstance(head_map, str):
return (head_map,)
return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
def _git(*args: str) -> str:
def _git(*args: str) -> str | None:
result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
return result.stdout if result.returncode == 0 else ""
return result.stdout if result.returncode == 0 else None
def snapshot(revision: str) -> Snapshot:
return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
diff: Final = _git("diff", "--name-only", "--no-renames", base, head)
return None if diff is None else tuple(diff.splitlines())
def main(argv: Sequence[str]) -> int:
@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int:
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
args: Final = parser.parse_args(argv)
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
changed: Final = changed_files(args.base, args.head)
if changed is None:
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
return 1
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
contract: Final = contract_for(bot, changed)
if failures:
print(f"cost map guard failed ({contract}):")
print("\n".join(f"- {failure}" for failure in failures))

View file

@ -108,6 +108,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/cursor/",
"/milvus/",
"/openai_passthrough/",
"/tinyfish/",
# Dynamic provider / toolset passthrough (path templates)
"/{provider}/",
"/toolset/",

View file

@ -2894,6 +2894,7 @@ dependencies = [
"litellm-host",
"litellm-http",
"litellm-llms",
"litellm-secrets",
"litellm-types",
"mime_guess",
"moka",
@ -3006,6 +3007,7 @@ dependencies = [
"litellm-framing",
"litellm-host",
"litellm-http",
"litellm-secrets",
"litellm-types",
"reqwest 0.12.28",
"rstest",
@ -3024,6 +3026,7 @@ dependencies = [
name = "litellm-python-bridge"
version = "0.1.0"
dependencies = [
"aws-sdk-secretsmanager",
"bytes",
"criterion",
"futures-util",
@ -3047,6 +3050,9 @@ dependencies = [
"litellm-host-python",
"litellm-http",
"litellm-llms",
"litellm-secrets",
"litellm-secrets-aws",
"litellm-secrets-types",
"litellm-token-counter",
"litellm-types",
"pyo3",
@ -3062,6 +3068,7 @@ dependencies = [
"tokio",
"tokio-tungstenite",
"url",
"wiremock",
]
[[package]]
@ -3246,6 +3253,7 @@ dependencies = [
name = "litellm-token-counter-huggingface"
version = "0.1.0"
dependencies = [
"serde_json",
"thiserror 2.0.19",
"tokenizers",
]
@ -3254,6 +3262,9 @@ dependencies = [
name = "litellm-token-counter-tiktoken"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"once_cell",
"rustc-hash",
"thiserror 2.0.19",
"tiktoken-rs",
]

View file

@ -89,7 +89,7 @@ veil = "0.3.0"
[profile.release]
opt-level = 3
lto = "thin"
lto = "fat"
codegen-units = 1
panic = "unwind"
debug = false

View file

@ -621,6 +621,26 @@ mod tests {
None
}
#[test]
fn secret_names_cover_environment_reads() {
let seen = std::sync::Arc::new(std::sync::Mutex::new(
std::collections::BTreeSet::<String>::new(),
));
let recorded = seen.clone();
let env = |name: &str| {
recorded.lock().unwrap().insert(name.to_string());
None
};
resolve_aws_region(None, &Map::new(), &env);
aws_auth_config(&Map::new(), &env);
assert!(
seen.lock()
.unwrap()
.iter()
.all(|name| crate::constants::SECRET_NAMES.contains(&name.as_str()))
);
}
#[test]
fn a_region_comes_from_the_call_then_the_model_then_the_environment() {
let params = Map::from_iter([("aws_region_name".to_string(), Value::from("eu-west-1"))]);

View file

@ -14,6 +14,19 @@ pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT";
pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID";
pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
pub const SECRET_NAMES: &[&str] = &[
AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY,
AWS_SESSION_TOKEN,
AWS_REGION_NAME,
AWS_REGION,
AWS_SESSION_NAME,
AWS_PROFILE_NAME,
AWS_ROLE_NAME,
AWS_WEB_IDENTITY_TOKEN,
AWS_STS_ENDPOINT,
AWS_EXTERNAL_ID,
];
/// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
/// Python's `_filter_headers_for_aws_signature` allowlist.

View file

@ -3,5 +3,5 @@ mod native;
mod resolve;
mod types;
pub use resolve::AzureAuthService;
pub use resolve::{AzureAuthService, SECRET_NAMES};
pub use types::{AzureAuthInputs, ConfigValue};

View file

@ -19,6 +19,17 @@ const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST";
const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL";
const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE";
pub const SECRET_NAMES: &[&str] = &[
AZURE_AD_TOKEN_ENV,
AZURE_TENANT_ID_ENV,
AZURE_CLIENT_ID_ENV,
AZURE_CLIENT_SECRET_ENV,
AZURE_SCOPE_ENV,
AZURE_AUTHORITY_HOST_ENV,
AZURE_CREDENTIAL_ENV,
AZURE_FEDERATED_TOKEN_FILE_ENV,
];
#[derive(Clone, Debug)]
pub(crate) enum AzureCredentialPlan {
Supplied(Sourced<ResolvedCredential>),
@ -440,13 +451,14 @@ fn non_empty_reference(value: &str, kind: &str) -> Result<String, Error> {
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use std::future::Future;
use std::sync::{Arc, Mutex};
use serde_json::json;
use super::{
AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference,
AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, SECRET_NAMES, oidc_reference,
resolve_reference, select_auth_plan,
};
use crate::native::ValidatedAzureRequest;
@ -517,6 +529,24 @@ mod tests {
assert!(matches!(plan, AzureCredentialPlan::Native(_)));
}
#[test]
fn secret_names_cover_environment_reads() {
let seen = std::sync::Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
let recorded = seen.clone();
let inputs = AzureAuthInputs::default();
select_auth_plan(&inputs, &|name| {
recorded.lock().unwrap().insert(name.to_string());
None
})
.unwrap();
assert!(
seen.lock()
.unwrap()
.iter()
.all(|name| SECRET_NAMES.contains(&name.as_str()))
);
}
#[test]
fn supplied_token_does_not_require_refresh() {
let params = json!({"azure_ad_token": "token"});

View file

@ -23,6 +23,16 @@ const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT";
const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION";
const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION";
pub const SECRET_NAMES: &[&str] = &[
VERTEX_AI_API_KEY_ENV,
VERTEXAI_API_KEY_ENV,
VERTEXAI_CREDENTIALS_ENV,
GOOGLE_APPLICATION_CREDENTIALS_ENV,
VERTEXAI_PROJECT_ENV,
VERTEXAI_LOCATION_ENV,
VERTEX_LOCATION_ENV,
];
#[derive(Clone, Debug, Default)]
pub struct VertexConfig {
credentials: Option<Sourced<SecretValue>>,
@ -406,6 +416,7 @@ fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
#[cfg(test)]
mod tests {
use std::collections::BTreeSet;
use std::sync::atomic::{AtomicUsize, Ordering};
use serde_json::json;
@ -476,6 +487,27 @@ mod tests {
);
}
#[tokio::test]
async fn secret_names_cover_environment_reads() {
let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
let recorded = seen.clone();
let env = |name: &str| {
recorded.lock().unwrap().insert(name.to_string());
None
};
let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
.await
.unwrap();
get_vertex_ai_location(&VertexConfig::default(), &env);
assert!(
seen.lock()
.unwrap()
.iter()
.all(|name| SECRET_NAMES.contains(&name.as_str()))
);
}
#[test]
fn empty_primary_values_fall_back_to_python_aliases() {
let config = config(json!({

View file

@ -1,4 +1,5 @@
use serde::Deserialize;
use std::hash::{Hash, Hasher};
use veil::Redact;
#[derive(Redact, Clone, Deserialize)]
@ -23,6 +24,12 @@ impl PartialEq for SecretValue {
impl Eq for SecretValue {}
impl Hash for SecretValue {
fn hash<H: Hasher>(&self, state: &mut H) {
self.0.hash(state);
}
}
#[cfg(test)]
mod tests {
use super::SecretValue;

View file

@ -36,6 +36,8 @@ Callers supply Unix time for response freshness. Backend TTL uses its own clock.
The extension keeps a private test harness for memory and Redis single and batch response lookup and storage. Batch lookup returns ordered values plus missing indices for embedding partial-hit wiring. No bridge-only cache type is part of the public API
The bridge also exposes a production-shaped response cache runtime selected through the Rust catalog. Its shipped rule set is empty, so current SDK, Router, and proxy calls stay on Python and do not construct native cache resources. Tests can inject a rule and build the native memory runtime from an ordinary Python `Cache` configuration without changing the legacy cache classes
Object responses are written as they are, and every other response shape is written as a serialized string, which is the pair of shapes Python reads. A string on the wire is therefore always a serialized response, so string-valued responses round trip. Typed backends such as memory never pass through the codec
The resolver reads the namespace's `cache` attribute each time it resolves. A captured binding retains the selected service for its operation, including background writes. `None` disables caching. Custom Python cache objects keep their original methods, arguments, returned awaitables, exceptions, and caller-task execution

View file

@ -1,9 +1,9 @@
use std::{sync::Mutex, time::Duration};
use litellm_cache::{BaseCache, Error, ExactCacheContext};
use litellm_cache::Error;
use serde_json::Value;
use crate::{CacheEntry, ResponseCache, ResponseCacheRequest};
use crate::{ExactResponseCache, ResponseCacheRequest};
pub struct WriteBuffer {
flush_size: usize,
@ -18,9 +18,9 @@ impl WriteBuffer {
}
}
pub async fn async_store<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>>(
pub async fn async_store(
&self,
cache: &ResponseCache<B>,
cache: &dyn ExactResponseCache,
request: &ResponseCacheRequest,
response: Value,
now: Duration,

View file

@ -0,0 +1,148 @@
use std::{future::Future, pin::Pin, time::Duration};
use litellm_cache::{
BaseCache, BatchCache, CacheConnectionResult, Error, ExactCacheContext, FlushCache,
};
use serde_json::Value;
use crate::{CacheEntry, PartialHits, ResponseCache, ResponseCacheRequest};
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
/// Object-safe view of a `ResponseCache` over an exact-match backend, so hosts can hold every
/// exact backend behind one pointer without erasing which backend it is elsewhere.
pub trait ExactResponseCache: Send + Sync {
fn default_ttl(&self) -> Option<Duration>;
fn lookup(&self, request: &ResponseCacheRequest, now: Duration)
-> Result<Option<Value>, Error>;
fn store(
&self,
request: &ResponseCacheRequest,
response: Value,
now: Duration,
) -> Result<(), Error>;
fn lookup_batch(
&self,
requests: &[ResponseCacheRequest],
now: Duration,
) -> Result<PartialHits, Error>;
fn async_lookup<'a>(
&'a self,
request: &'a ResponseCacheRequest,
now: Duration,
) -> BoxFuture<'a, Result<Option<Value>, Error>>;
fn async_store<'a>(
&'a self,
request: &'a ResponseCacheRequest,
response: Value,
now: Duration,
) -> BoxFuture<'a, Result<(), Error>>;
fn async_lookup_batch<'a>(
&'a self,
requests: &'a [ResponseCacheRequest],
now: Duration,
) -> BoxFuture<'a, Result<PartialHits, Error>>;
fn async_store_batch<'a>(
&'a self,
entries: Vec<(ResponseCacheRequest, Value)>,
now: Duration,
) -> BoxFuture<'a, Result<(), Error>>;
fn async_store_entries<'a>(
&'a self,
entries: Vec<(ResponseCacheRequest, Value, Duration)>,
) -> BoxFuture<'a, Result<(), Error>>;
fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>;
fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result<CacheConnectionResult, Error>>;
}
impl<B> ExactResponseCache for ResponseCache<B>
where
B: BaseCache<Value = CacheEntry, Context = ExactCacheContext> + BatchCache + FlushCache,
{
fn default_ttl(&self) -> Option<Duration> {
ResponseCache::default_ttl(self)
}
fn lookup(
&self,
request: &ResponseCacheRequest,
now: Duration,
) -> Result<Option<Value>, Error> {
ResponseCache::lookup(self, request, now)
}
fn store(
&self,
request: &ResponseCacheRequest,
response: Value,
now: Duration,
) -> Result<(), Error> {
ResponseCache::store(self, request, response, now)
}
fn lookup_batch(
&self,
requests: &[ResponseCacheRequest],
now: Duration,
) -> Result<PartialHits, Error> {
ResponseCache::lookup_batch(self, requests, now)
}
fn async_lookup<'a>(
&'a self,
request: &'a ResponseCacheRequest,
now: Duration,
) -> BoxFuture<'a, Result<Option<Value>, Error>> {
Box::pin(ResponseCache::async_lookup(self, request, now))
}
fn async_store<'a>(
&'a self,
request: &'a ResponseCacheRequest,
response: Value,
now: Duration,
) -> BoxFuture<'a, Result<(), Error>> {
Box::pin(ResponseCache::async_store(self, request, response, now))
}
fn async_lookup_batch<'a>(
&'a self,
requests: &'a [ResponseCacheRequest],
now: Duration,
) -> BoxFuture<'a, Result<PartialHits, Error>> {
Box::pin(ResponseCache::async_lookup_batch(self, requests, now))
}
fn async_store_batch<'a>(
&'a self,
entries: Vec<(ResponseCacheRequest, Value)>,
now: Duration,
) -> BoxFuture<'a, Result<(), Error>> {
Box::pin(ResponseCache::async_store_batch(self, entries, now))
}
fn async_store_entries<'a>(
&'a self,
entries: Vec<(ResponseCacheRequest, Value, Duration)>,
) -> BoxFuture<'a, Result<(), Error>> {
Box::pin(ResponseCache::async_store_entries(self, entries))
}
fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>> {
Box::pin(ResponseCache::async_flush(self))
}
fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result<CacheConnectionResult, Error>> {
Box::pin(ResponseCache::test_connection(self))
}
}

View file

@ -2,6 +2,7 @@ mod buffer;
mod caching;
mod codec;
mod embedding;
mod exact;
mod response;
pub use buffer::WriteBuffer;
@ -11,4 +12,5 @@ pub use caching::{
};
pub use codec::ResponseCacheCodec;
pub use embedding::PartialHits;
pub use exact::ExactResponseCache;
pub use response::{ResponseCache, ResponseCacheRequest};

View file

@ -17,6 +17,11 @@ pub fn parse_str_bool(value: &str) -> Option<bool> {
token.eq_ignore_ascii_case("false").then_some(false)
}
/// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true.
pub fn parse_redis_bool(value: &str) -> bool {
value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes")
}
impl<'de> DeserializeAs<'de, i64> for LaxI64 {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
deserializer.deserialize_any(Self)

View file

@ -36,6 +36,7 @@ url.workspace = true
veil.workspace = true
[dev-dependencies]
litellm-secrets.workspace = true
litellm-auth-gcp.workspace = true
litellm-llms = { workspace = true, features = ["test-support"] }
rstest.workspace = true

View file

@ -22,7 +22,12 @@ pub(crate) async fn perform_ocr_request(
) -> Result<LiteLLMOcrResponse, Error> {
request.response_format()?;
let config = request.config;
let request = prepare_request(request, caller_document, client);
let secrets = client
.secret_source()
.resolve(&config.secret_names())
.await
.map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
let request = prepare_request(request, caller_document, client, secrets);
let hooks = OcrCallHooks::new(host.clone(), &request, config);
config.ocr(client, &request, &hooks).await
}

View file

@ -1,7 +1,10 @@
use litellm_auth::{InputSource, SecretValue, Sourced};
use litellm_llms::base_llm::ocr::{
handler::OcrClient,
transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
use litellm_llms::base_llm::{
inference::secrets::Secrets,
ocr::{
handler::OcrClient,
transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
},
};
use super::provider_config::OcrProvider;
@ -11,6 +14,7 @@ pub(crate) fn prepare_request(
request: ResolvedOcrRequest,
caller_document: bool,
client: &OcrClient,
secrets: Secrets,
) -> PreparedOcrRequest {
let credentials = request.credentials.clone();
let (preferred_api_key_env, api_base_env) = match request.config.provider() {
@ -24,7 +28,7 @@ pub(crate) fn prepare_request(
| OcrProvider::Reducto
| OcrProvider::VertexAi => (None, None),
};
let secret = |name: &str| client.secrets().truthy(name);
let secret = |name: &str| secrets.truthy(name);
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
credentials.api_key.clone().or_else(|| {
preferred_api_key_env
@ -60,12 +64,7 @@ pub(crate) fn prepare_request(
PreparedOcrRequest {
model,
document,
connection: OcrConnection::new(
resolved,
transport,
client.settings().clone(),
client.secrets().clone(),
),
connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
caller_document,
optional_params,
input_sources,
@ -79,6 +78,7 @@ pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedO
request,
true,
&OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()),
std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
)
}

View file

@ -114,6 +114,10 @@ impl OcrConfigKind {
with_config!(self, config => config.get_api_key_env_var())
}
pub(crate) fn secret_names(self) -> Vec<&'static str> {
with_config!(self, config => config.secret_names())
}
pub(crate) fn get_health_check_document(self) -> OcrDocument {
with_config!(self, config => config.get_health_check_document())
}
@ -213,6 +217,8 @@ fn is_document_intelligence_model(model: &str) -> bool {
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use litellm_auth::{InputSource, Sourced};
use litellm_llms::{
base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
@ -221,6 +227,27 @@ mod tests {
use super::*;
#[rstest]
#[case(OcrConfigKind::AwsTextract)]
#[case(OcrConfigKind::AwsTextractAnalyze)]
#[case(OcrConfigKind::Cohere)]
#[case(OcrConfigKind::Mistral)]
#[case(OcrConfigKind::AzureAi)]
#[case(OcrConfigKind::AzureCohere)]
#[case(OcrConfigKind::AzureDocumentIntelligence)]
#[case(OcrConfigKind::ReductoLegacy)]
#[case(OcrConfigKind::ReductoV3)]
#[case(OcrConfigKind::VertexAi)]
#[case(OcrConfigKind::VertexDeepSeek)]
fn secret_names_include_api_keys_without_duplicates(#[case] config: OcrConfigKind) {
let names = config.secret_names();
let unique = names.iter().collect::<HashSet<_>>();
assert_eq!(names.len(), unique.len());
if let Some(api_key) = config.get_api_key_env_var() {
assert!(names.contains(&api_key));
}
}
#[rstest]
#[case("cohere")]
#[case("mistral")]

View file

@ -1,5 +1,6 @@
use std::sync::{Arc, Mutex};
use futures_util::future::BoxFuture;
use litellm_auth_gcp::VertexAuth;
use litellm_host::{
event::{CallEvent, MachineEvent, WireRequest},
@ -10,11 +11,14 @@ use litellm_http::{
HttpClientPool, HttpSettings, Resolution,
media::{PublicDnsResolver, UrlPolicy},
};
use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets};
use litellm_llms::base_llm::ocr::{
error::Error as OcrError,
handler::OcrClient,
settings::OcrSettings,
transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig},
transformation::{
BaseOcrConfig, LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig,
},
};
use rstest::rstest;
use serde_json::{Value, json};
@ -27,6 +31,32 @@ use super::{
};
use crate::ocr::route::{LocalOcrHost, OcrOp, OcrOpResult, ocr_machine};
struct RecordingSecretSource {
names: Arc<Mutex<Vec<&'static str>>>,
values: &'static [(&'static str, &'static str)],
api_base: String,
}
impl SecretSource for RecordingSecretSource {
fn resolve<'a>(
&'a self,
names: &'a [&'static str],
) -> BoxFuture<'a, Result<Secrets, litellm_secrets::Error>> {
*self.names.lock().unwrap() = names.to_vec();
let values = self.values;
let api_base = self.api_base.clone();
Box::pin(async move {
Ok(Arc::new(move |name: &str| match name {
"MISTRAL_AZURE_API_BASE" => Some(api_base.clone()),
_ => values
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string()),
}) as Secrets)
})
}
}
#[rstest]
#[case::mistral("mistral/model", json!({}))]
#[case::vertex("vertex_ai/mistral-ocr-latest", json!({"vertex_project":"test-project", "vertex_location":"us-central1"}))]
@ -184,14 +214,11 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source(
#[case] expected_key: &str,
) {
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
let secret_base = base.clone();
let client = ocr_client().with_secrets(Arc::new(move |name: &str| match name {
"MISTRAL_AZURE_API_BASE" => Some(secret_base.clone()),
"MISTRAL_API_BASE" => Some("http://127.0.0.1:9/never-read".into()),
_ => secrets
.iter()
.find(|(key, _)| *key == name)
.map(|(_, value)| value.to_string()),
let names = Arc::new(Mutex::new(Vec::new()));
let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource {
names: names.clone(),
values: secrets,
api_base: base.clone(),
}));
let request = decode_request(OcrWireRequest {
model: "mistral/model".into(),
@ -208,9 +235,47 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source(
crate::ocr::client::perform(&client, request).await.unwrap();
server.await.unwrap();
assert_eq!(
*names.lock().unwrap(),
litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names()
);
assert!(seen.lock().unwrap()[0].contains(&format!("authorization: Bearer {expected_key}")));
}
#[tokio::test]
async fn mistral_ocr_resolves_provider_secrets_before_transformation() {
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
let names = Arc::new(Mutex::new(Vec::new()));
let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource {
names: names.clone(),
values: &[("MISTRAL_API_KEY", "source-key")],
api_base: base.clone(),
}));
let request = decode_request(OcrWireRequest {
model: "mistral/mistral-ocr-latest".into(),
document: json!({
"type":"document_url",
"document_url":"data:application/pdf;base64,YWJj"
}),
api_key: None,
api_base: None,
custom_llm_provider: None,
extra_headers: None,
optional_params: Default::default(),
input_sources: Default::default(),
timeout_seconds: Some(2.0),
})
.unwrap();
crate::ocr::client::perform(&client, request).await.unwrap();
server.await.unwrap();
assert_eq!(
*names.lock().unwrap(),
litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names()
);
assert!(seen.lock().unwrap()[0].contains("authorization: Bearer source-key"));
}
#[tokio::test]
async fn ocr_client_uses_the_injected_http_pool_configuration() {
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
@ -224,7 +289,7 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() {
UrlPolicy::default(),
VertexAuth::default(),
OcrSettings::default(),
Arc::new(litellm_core_utils::settings::ProcessEnvironment),
Arc::new(litellm_llms::base_llm::inference::secrets::EnvironmentSecrets),
)
.unwrap();
crate::ocr::client::perform(&client, wire_request("mistral/model", &base, json!({})))

View file

@ -29,7 +29,7 @@ pyo3::create_exception!(
static FORK_GATE: ForkGate = ForkGate::new();
/// Whether this process has started the Tokio runtime.
/// Whether this process has entered process-bound native execution.
pub fn runtime_started() -> bool {
FORK_GATE.started(std::process::id())
}
@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> {
FORK_GATE.reserve(std::process::id())
}
/// The only door to the Tokio runtime: every route reaches it through this module, which is
/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it.
fn enter_runtime() -> PyResult<()> {
/// Claims process-bound native state before runtime startup or tokenizer execution.
pub fn enter_native() -> PyResult<()> {
FORK_GATE
.enter(std::process::id())
.map_err(|refused| match refused {
@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> {
#[expect(clippy::disallowed_methods, reason = "this is the gated door")]
fn runtime() -> PyResult<&'static Runtime> {
enter_runtime()?;
enter_native()?;
Ok(pyo3_async_runtimes::tokio::get_runtime())
}
@ -70,7 +69,7 @@ where
F: Future<Output = PyResult<T>> + Send + 'static,
T: for<'py> IntoPyObject<'py> + Send + 'static,
{
enter_runtime()?;
enter_native()?;
pyo3_async_runtimes::tokio::future_into_py(py, future)
}

View file

@ -20,7 +20,7 @@ pub use argument::lookup;
pub use callable::wrap_failure;
pub use driver::run_call;
pub use execution::{
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value,
reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value,
runtime_started,
};

View file

@ -18,6 +18,7 @@ litellm-auth-gcp.workspace = true
litellm-host.workspace = true
litellm-framing.workspace = true
litellm-http.workspace = true
litellm-secrets.workspace = true
base64.workspace = true
bytes.workspace = true
data-url = "0.3.2"

View file

@ -40,6 +40,10 @@ impl BaseOcrConfig for TextractAnalyzeDocumentConfig {
type ProviderRequest = AnalyzeDocumentRequest;
type Environment = TextractEnvironment;
fn secret_names(&self) -> Vec<&'static str> {
litellm_auth_aws::constants::SECRET_NAMES.to_vec()
}
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
&["feature_types"]
}

View file

@ -29,6 +29,10 @@ impl BaseOcrConfig for TextractDetectTextConfig {
type ProviderRequest = DetectDocumentTextRequest;
type Environment = TextractEnvironment;
fn secret_names(&self) -> Vec<&'static str> {
litellm_auth_aws::constants::SECRET_NAMES.to_vec()
}
fn get_health_check_document(&self) -> OcrDocument {
health_check_document()
}

View file

@ -28,6 +28,10 @@ impl BaseOcrConfig for AzureAICohereParseConfig {
super::transformation::AzureAiOcrConfig.get_api_key_env_var()
}
fn secret_names(&self) -> Vec<&'static str> {
super::transformation::AzureAiOcrConfig.secret_names()
}
fn get_health_check_document(&self) -> OcrDocument {
CohereParseConfig.get_health_check_document()
}

View file

@ -2,7 +2,7 @@ use std::{collections::BTreeSet, time::Duration};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_auth::{InputSource, Sourced};
use litellm_auth_azure::AzureAuthInputs;
use litellm_auth_azure::{AzureAuthInputs, SECRET_NAMES as AZURE_AUTH_SECRET_NAMES};
use litellm_core_utils::{
call_arguments::CallArguments,
serde_compat::{FiniteF64, LaxI64},
@ -141,6 +141,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
Some(AZURE_DI_API_KEY_ENV)
}
fn secret_names(&self) -> Vec<&'static str> {
[
[AZURE_DI_API_KEY_ENV, AZURE_DI_ENDPOINT_ENV].as_slice(),
AZURE_AUTH_SECRET_NAMES,
]
.into_iter()
.flatten()
.copied()
.collect()
}
fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials {
ResolvedOcrCredentials {
api_key: inputs.api_key.and_then(|key| {

View file

@ -1,5 +1,6 @@
use litellm_auth::{InputSource, Sourced};
use litellm_auth_azure::AzureAuthInputs;
use litellm_auth_azure::SECRET_NAMES as AZURE_AUTH_SECRET_NAMES;
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
use serde_json::Value;
@ -37,6 +38,17 @@ impl BaseOcrConfig for AzureAiOcrConfig {
Some(AZURE_AI_API_KEY_ENV)
}
fn secret_names(&self) -> Vec<&'static str> {
[
[AZURE_AI_API_KEY_ENV, AZURE_AI_API_BASE_ENV].as_slice(),
AZURE_AUTH_SECRET_NAMES,
]
.into_iter()
.flatten()
.copied()
.collect()
}
fn map_ocr_params(
&self,
non_default_params: &CallArguments,

View file

@ -0,0 +1 @@
pub mod secrets;

View file

@ -0,0 +1,19 @@
use std::sync::Arc;
use futures_util::future::BoxFuture;
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
use litellm_secrets::Error;
pub type Secrets = Arc<dyn Lookup + Send + Sync>;
pub trait SecretSource: Send + Sync {
fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result<Secrets, Error>>;
}
pub struct EnvironmentSecrets;
impl SecretSource for EnvironmentSecrets {
fn resolve<'a>(&'a self, _names: &'a [&'static str]) -> BoxFuture<'a, Result<Secrets, Error>> {
Box::pin(async { Ok(Arc::new(ProcessEnvironment) as Secrets) })
}
}

View file

@ -2,5 +2,6 @@ pub mod anthropic_messages;
pub mod audio_transcription;
pub mod base_model_iterator;
pub mod chat;
pub mod inference;
pub mod ocr;
pub mod responses;

View file

@ -98,6 +98,8 @@ pub enum Error {
"Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()"
)]
MissingReductoApiKey,
#[error("secret resolution failed: {0}")]
Secret(#[source] std::sync::Arc<litellm_secrets::Error>),
#[error(transparent)]
Auth(#[from] litellm_auth::Error),
#[error(transparent)]

View file

@ -1,3 +1,5 @@
use std::sync::Arc;
use bytes::{Bytes, BytesMut};
use futures_util::future::BoxFuture;
use litellm_auth_gcp::VertexAuth;
@ -11,9 +13,10 @@ use litellm_http::{
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
use crate::base_llm::inference::secrets::SecretSource;
use crate::base_llm::ocr::{
error::Error,
settings::{OcrSettings, Secrets},
settings::OcrSettings,
transformation::{
BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext,
PreparedOcrRequest, decode_request_value, decode_response,
@ -35,7 +38,7 @@ pub struct OcrClient {
document_fetcher: MediaFetcher,
vertex_auth: VertexAuth,
settings: OcrSettings,
secrets: Secrets,
secrets: Arc<dyn SecretSource>,
}
impl OcrClient {
@ -45,7 +48,7 @@ impl OcrClient {
url_policy: UrlPolicy,
vertex_auth: VertexAuth,
settings: OcrSettings,
secrets: Secrets,
secrets: Arc<dyn SecretSource>,
) -> Result<Self, litellm_http::Error> {
Ok(Self {
provider_http: pool.client(config, ClientVariant::Provider)?,
@ -77,7 +80,7 @@ impl OcrClient {
&self.settings
}
pub fn secrets(&self) -> &Secrets {
pub fn secret_source(&self) -> &Arc<dyn SecretSource> {
&self.secrets
}
@ -92,7 +95,7 @@ impl OcrClient {
document_fetcher: MediaFetcher::for_test(document_http),
vertex_auth: VertexAuth::default(),
settings: OcrSettings::default(),
secrets: std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
secrets: Arc::new(crate::base_llm::inference::secrets::EnvironmentSecrets),
}
}
@ -102,7 +105,7 @@ impl OcrClient {
}
#[cfg(any(test, feature = "test-support"))]
pub fn with_secrets(self, secrets: Secrets) -> Self {
pub fn with_secrets(self, secrets: Arc<dyn SecretSource>) -> Self {
Self { secrets, ..self }
}
}

View file

@ -1,9 +1,7 @@
use std::{sync::Arc, time::Duration};
use std::time::Duration;
use litellm_core_utils::settings::Lookup;
pub type Secrets = Arc<dyn Lookup + Send + Sync>;
#[derive(Clone, Debug, PartialEq)]
pub struct OcrSettings {
pub request_timeout: Duration,

View file

@ -14,10 +14,13 @@ use serde::{
use serde_json::{Map, Value};
use serde_with::serde_as;
use crate::base_llm::ocr::{
error::Error,
handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body},
settings::{OcrSettings, Secrets},
use crate::base_llm::{
inference::secrets::Secrets,
ocr::{
error::Error,
handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body},
settings::OcrSettings,
},
};
pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
@ -436,6 +439,8 @@ pub trait BaseOcrConfig: Send + Sync + Sized + 'static {
None
}
fn secret_names(&self) -> Vec<&'static str>;
fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials {
ResolvedOcrCredentials {
api_key: inputs

View file

@ -102,6 +102,10 @@ impl BaseOcrConfig for CohereParseConfig {
Some(COHERE_API_KEY_ENV)
}
fn secret_names(&self) -> Vec<&'static str> {
vec![COHERE_API_KEY_ENV]
}
fn get_health_check_document(&self) -> OcrDocument {
OcrDocument::ImageUrl {
image_url: COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI.into(),

View file

@ -69,6 +69,14 @@ impl BaseOcrConfig for MistralOcrConfig {
Some(MISTRAL_OCR_API_KEY_ENV_VAR)
}
fn secret_names(&self) -> Vec<&'static str> {
vec![
MISTRAL_OCR_API_KEY_ENV_VAR,
"MISTRAL_AZURE_API_KEY",
"MISTRAL_AZURE_API_BASE",
]
}
fn map_ocr_params(
&self,
non_default_params: &CallArguments,

View file

@ -92,6 +92,10 @@ impl BaseOcrConfig for ReductoParseV3Config {
type ProviderRequest = ReductoV3Request;
type Environment = Vec<(String, String)>;
fn secret_names(&self) -> Vec<&'static str> {
vec![REDUCTO_API_KEY_ENV]
}
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
&["formatting", "retrieval", "settings"]
}
@ -180,6 +184,10 @@ impl BaseOcrConfig for ReductoParseLegacyConfig {
type ProviderRequest = ReductoLegacyRequest;
type Environment = Vec<(String, String)>;
fn secret_names(&self) -> Vec<&'static str> {
vec![REDUCTO_API_KEY_ENV]
}
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
&["enhance"]
}

View file

@ -105,6 +105,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig {
VertexAiOcrConfig.get_api_key_env_var()
}
fn secret_names(&self) -> Vec<&'static str> {
VertexAiOcrConfig.secret_names()
}
fn map_ocr_params(
&self,
_arguments: &CallArguments,

View file

@ -34,6 +34,10 @@ impl BaseOcrConfig for VertexAiOcrConfig {
Some("VERTEX_AI_API_KEY")
}
fn secret_names(&self) -> Vec<&'static str> {
litellm_auth_gcp::SECRET_NAMES.to_vec()
}
fn map_ocr_params(
&self,
non_default_params: &CallArguments,

View file

@ -10,7 +10,7 @@ name = "_native"
crate-type = ["cdylib"]
[features]
default = ["abi3", "fast"]
default = ["abi3", "fast", "huggingface", "tiktoken"]
abi3 = ["pyo3/abi3-py310"]
extension-module = ["pyo3/extension-module"]
panic-test = []
@ -20,6 +20,7 @@ tiktoken = ["litellm-token-counter/tiktoken"]
[dependencies]
bytes.workspace = true
futures-util.workspace = true
litellm-cache.workspace = true
litellm-cache-azure-blob.workspace = true
litellm-cache-memory.workspace = true
@ -41,6 +42,8 @@ litellm-core-utils.workspace = true
litellm-auth-gcp.workspace = true
litellm-http.workspace = true
litellm-llms.workspace = true
litellm-secrets = { workspace = true, features = ["aws"] }
litellm-secrets-types.workspace = true
litellm-types.workspace = true
litellm-host-python.workspace = true
litellm-token-counter = { path = "../token-counter", default-features = false }
@ -53,6 +56,7 @@ url.workspace = true
tokio = { workspace = true, features = ["rt", "sync"] }
[dev-dependencies]
litellm-secrets-aws.workspace = true
serde.workspace = true
serde_with.workspace = true
criterion.workspace = true
@ -60,6 +64,8 @@ futures-util.workspace = true
rstest.workspace = true
sha2.workspace = true
tokio-tungstenite.workspace = true
wiremock = "0.6.5"
aws-sdk-secretsmanager = "1.117.0"
[[bench]]
name = "serialization"

View file

@ -0,0 +1,5 @@
Native OCR uses `SecretSource` with `EnvironmentSecrets`, preserving process-environment reads. Readable Python secret managers still make OCR decline to the existing Python implementation. `ResolvedSecrets` and the separate `secret_manager_binding()` snapshot are inactive foundations for a later rollout
Cache and secret-manager catalog entries remain Python-only, including when `LITELLM_RUST=1`. The new cache runtime is not connected to SDK or gateway caching
OCR provider requests use the shared `litellm-http` pool. AWS and Google secret-manager SDK clients keep their SDK transports, which do not yet inherit the pool's proxy, TLS, certificate, timeout, or observability configuration. Preserve those SDK transports and configure them equivalently instead of forcing them through reqwest

View file

@ -1,154 +0,0 @@
{
"http_settings": {
"version": 1,
"fields": {
"ssl_verify": {
"adapter": "SslVerifyInput",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [
"none",
"bool",
"str"
],
"unsupported_live": "configuration_error"
},
"ssl_certificate": {
"adapter": "OptionalStrictString",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"ssl_security_level": {
"adapter": "TuningString",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"ssl_ecdh_curve": {
"adapter": "TuningString",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"force_ipv4": {
"adapter": "Truthy",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"http2": {
"adapter": "ExactTrue",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"aiohttp_trust_env": {
"adapter": "Truthy",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"disable_aiohttp_trust_env": {
"adapter": "Truthy",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"disable_aiohttp_transport": {
"adapter": "ExactTrue",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"user_agent": {
"adapter": "StrictString",
"required": true,
"precedence": "accessor",
"sensitive": false,
"shapes": [],
"unsupported_live": null
}
}
},
"url_policy": {
"version": 1,
"fields": {
"user_url_validation": {
"adapter": "Truthy",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
},
"user_url_allowed_hosts": {
"adapter": "HostCollection",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
}
}
},
"provider_defaults": {
"version": 1,
"fields": {
"vertex_project": {
"adapter": "FalsyOptionalString",
"required": true,
"precedence": "module_global",
"sensitive": true,
"shapes": [],
"unsupported_live": null
},
"vertex_location": {
"adapter": "FalsyOptionalString",
"required": true,
"precedence": "module_global",
"sensitive": true,
"shapes": [],
"unsupported_live": null
},
"enable_azure_ad_token_refresh": {
"adapter": "ExactTrue",
"required": true,
"precedence": "module_global",
"sensitive": false,
"shapes": [],
"unsupported_live": null
}
}
},
"secret_manager": {
"version": 1,
"fields": {
"readable": {
"adapter": "StrictBool",
"required": true,
"precedence": "accessor",
"sensitive": false,
"shapes": [],
"unsupported_live": null
}
}
}
}

View file

@ -11,10 +11,12 @@ use serde_json::Value;
use super::{
cache_error,
callback::PythonCallback,
config::{CacheBackendConfig, CacheConfigProjection, NativeCacheConfig},
future::{ready_none, ready_value},
native::NativeResponseCache,
request::{now, request, requests},
};
use crate::errors::RustBridgeDeclined;
pub(super) enum CacheBinding {
Disabled,
@ -22,7 +24,7 @@ pub(super) enum CacheBinding {
PythonCallback(PythonCallback),
}
#[pyclass(frozen, name = "_CacheTestBinding")]
#[pyclass(frozen, name = "_ResponseCacheRuntime")]
pub(crate) struct ResolvedCache {
binding: CacheBinding,
pid: u32,
@ -66,6 +68,33 @@ impl ResolvedCache {
#[pymethods]
impl ResolvedCache {
#[staticmethod]
fn from_cache(cache: &Bound<'_, PyAny>) -> PyResult<Self> {
let config = match NativeCacheConfig::project(cache)? {
CacheConfigProjection::Native(config) => *config,
CacheConfigProjection::Unsupported(reason) => {
return Err(RustBridgeDeclined::new_err(reason.message()));
}
};
let service = match config.backend {
CacheBackendConfig::Memory(memory) => NativeResponseCache::memory(
memory.capacity,
memory.default_ttl,
memory.max_entry_bytes,
),
_ => {
return Err(RustBridgeDeclined::new_err(
"native response cache activation is not implemented for this backend",
));
}
};
Ok(Self::new(CacheBinding::Native(
service
.with_scope(config.policy.semantic_cache_scope)
.with_redis_flush_size(config.policy.redis_flush_size),
)))
}
#[getter]
fn kind(&self) -> &'static str {
match self.binding {

View file

@ -11,7 +11,7 @@ use pyo3::{
types::{PyAny, PyBool, PyDict, PyList, PyString},
};
use super::{native::NativeResponseCache, request::duration};
use super::{identity::BackendIdentity, native::NativeResponseCache, request::duration};
#[allow(dead_code, reason = "consumed by the cache activation follow-up")]
pub(super) struct CachePolicy {
@ -293,161 +293,58 @@ impl NativeCacheConfig {
}
pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
let default_ttl = match &self.backend {
CacheBackendConfig::Memory(config) => Some(config.default_ttl),
CacheBackendConfig::Redis(config) => Some(config.default_ttl),
CacheBackendConfig::S3(_) => None,
CacheBackendConfig::ValkeySemantic(_) => Some(Duration::ZERO),
CacheBackendConfig::Disk(_)
| CacheBackendConfig::AzureBlob(_)
| CacheBackendConfig::Gcs(_)
| CacheBackendConfig::RedisSemantic(_)
| CacheBackendConfig::QdrantSemantic(_) => None,
};
if !matches!(self.backend, CacheBackendConfig::ValkeySemantic(_))
&& service.default_ttl() != default_ttl
{
return Some("facade and native backend default TTLs must match");
}
match &self.backend {
CacheBackendConfig::Memory(config) if service.kind() != "memory" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::Memory(config) if service.capacity() != Some(config.capacity) => {
Some("facade and native backend capacities must match")
}
CacheBackendConfig::Memory(config)
if service.max_entry_bytes() != Some(config.max_entry_bytes) =>
{
Some("facade and native backend item limits must match")
}
CacheBackendConfig::Memory(_) => None,
CacheBackendConfig::Redis(_) if service.kind() != "redis" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::Redis(config) if service.topology() != Some(&config.topology) => {
Some("facade and native backend topologies must match")
}
CacheBackendConfig::Redis(config) => (service.namespace()
!= config.namespace.as_deref())
.then_some("facade and native backend namespaces must match"),
CacheBackendConfig::S3(_) if service.kind() != "s3" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::S3(config) if service.bucket() != Some(config.bucket.as_str()) => {
Some("facade and native backend buckets must match")
}
CacheBackendConfig::S3(config)
if service.key_prefix() != Some(config.key_prefix.as_str()) =>
{
Some("facade and native backend key prefixes must match")
}
CacheBackendConfig::S3(config) if service.region() != Some(config.region.as_str()) => {
Some("facade and native backend regions must match")
}
CacheBackendConfig::S3(config)
if service.endpoint()
!= config
.endpoint
.as_ref()
.map(|endpoint| endpoint.url.as_str()) =>
{
Some("facade and native backend endpoints must match")
}
CacheBackendConfig::S3(_) => None,
CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::Gcs(config)
if service
.gcs_backend()
.is_none_or(|backend| backend.bucket_name() != config.bucket_name) =>
{
Some("facade and native backend buckets must match")
}
CacheBackendConfig::Gcs(config)
if service
.gcs_backend()
.is_none_or(|backend| backend.key_prefix() != config.key_prefix) =>
{
Some("facade and native backend key prefixes must match")
}
CacheBackendConfig::Gcs(config)
if service.gcs_backend().is_none_or(|backend| {
backend.path_service_account() != config.path_service_account.as_deref()
}) =>
{
Some("facade and native backend credentials must match")
}
CacheBackendConfig::Gcs(_) => None,
CacheBackendConfig::ValkeySemantic(config) => {
if service.kind() != "valkey-semantic" {
return Some("facade and native backend types must match");
}
let Some((threshold, index_name)) = service.semantic_config() else {
return Some("facade and native backend types must match");
};
(threshold != config.similarity_threshold || index_name != config.index_name)
.then_some("facade and native semantic settings must match")
}
CacheBackendConfig::Disk(_) if service.kind() != "disk" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::Disk(config) => {
let Some(directory) = service.directory() else {
return Some("facade and native backend types must match");
};
let native = std::fs::canonicalize(directory).ok();
let facade = std::fs::canonicalize(&config.directory).ok();
(native != facade).then_some("facade and native backend directories must match")
}
CacheBackendConfig::RedisSemantic(_) if service.kind() != "redis_semantic" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::RedisSemantic(config)
if service.index_name() != Some(config.index_name.as_str()) =>
{
Some("facade and native backend index names must match")
}
CacheBackendConfig::RedisSemantic(config)
if service.similarity_threshold()
!= Some(f64::from(config.similarity_threshold as f32)) =>
{
Some("facade and native backend similarity thresholds must match")
}
CacheBackendConfig::RedisSemantic(_) => None,
CacheBackendConfig::QdrantSemantic(config) if service.kind() != "qdrant_semantic" => {
Some("facade and native backend types must match")
}
CacheBackendConfig::QdrantSemantic(config)
if service.collection_name() != Some(config.collection_name.as_str()) =>
{
Some("facade and native backend collections must match")
}
CacheBackendConfig::QdrantSemantic(config)
if service.similarity_threshold() != Some(config.similarity_threshold) =>
{
Some("facade and native backend similarity thresholds must match")
}
CacheBackendConfig::QdrantSemantic(config)
if service.vector_size() != Some(config.vector_size) =>
{
Some("facade and native backend vector sizes must match")
}
CacheBackendConfig::QdrantSemantic(config)
if service.embedding_model() != Some(config.embedding.model.as_str()) =>
{
Some("facade and native backend embedding models must match")
}
CacheBackendConfig::QdrantSemantic(_) => None,
CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() {
None => Some("facade and native backend types must match"),
Some((account_url, container))
if account_url != config.account_url || container != config.container =>
{
Some("facade and native backend containers must match")
}
Some(_) => None,
self.backend.identity().mismatch(&service.identity())
}
}
impl CacheBackendConfig {
/// The identity a native backend must have for this facade configuration to describe it.
pub(super) fn identity(&self) -> BackendIdentity {
match self {
Self::Memory(config) => BackendIdentity::Memory {
capacity: config.capacity,
max_entry_bytes: Some(config.max_entry_bytes),
default_ttl: Some(config.default_ttl),
},
Self::Redis(config) => BackendIdentity::Redis {
topology: config.topology.clone(),
namespace: config.namespace.clone(),
default_ttl: Some(config.default_ttl),
},
Self::S3(config) => BackendIdentity::S3 {
bucket: config.bucket.clone(),
key_prefix: config.key_prefix.clone(),
region: config.region.clone(),
endpoint: config
.endpoint
.as_ref()
.map(|endpoint| endpoint.url.clone()),
},
Self::Gcs(config) => BackendIdentity::Gcs {
bucket_name: config.bucket_name.clone(),
key_prefix: config.key_prefix.clone(),
path_service_account: config.path_service_account.clone(),
},
Self::ValkeySemantic(config) => BackendIdentity::ValkeySemantic {
index_name: config.index_name.clone(),
similarity_threshold: config.similarity_threshold,
},
Self::Disk(config) => BackendIdentity::Disk {
directory: config.directory.clone(),
},
Self::AzureBlob(config) => BackendIdentity::AzureBlob {
account_url: config.account_url.clone(),
container: config.container.clone(),
},
Self::RedisSemantic(config) => BackendIdentity::RedisSemantic {
index_name: config.index_name.clone(),
similarity_threshold: config.similarity_threshold as f32,
},
Self::QdrantSemantic(config) => BackendIdentity::QdrantSemantic {
collection_name: config.collection_name.clone(),
similarity_threshold: config.similarity_threshold,
vector_size: config.vector_size,
embedding_model: config.embedding.model.clone(),
},
}
}

View file

@ -9,6 +9,8 @@ tokio::task_local! {
static PREPARED_EMBEDDING: Result<Vec<f32>, Error>;
}
/// Runs `future` with the vector the Python embedder already produced, so the backend's
/// `async_embed` never has to call back into Python from the runtime.
pub(super) fn with_prepared_embedding<F: Future>(
vector: Result<Vec<f32>, Error>,
future: F,
@ -16,6 +18,7 @@ pub(super) fn with_prepared_embedding<F: Future>(
PREPARED_EMBEDDING.scope(vector, future)
}
/// The Python object that owns embedding for a semantic backend.
pub(super) struct PythonEmbedder(Py<PyAny>);
impl Clone for PythonEmbedder {
@ -29,10 +32,6 @@ impl PythonEmbedder {
Self(object)
}
pub(super) fn from_backend(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
Ok(Self(backend.clone().unbind()))
}
pub(super) fn object(&self) -> &Py<PyAny> {
&self.0
}
@ -41,18 +40,6 @@ impl PythonEmbedder {
visit.call(&self.0)
}
pub(super) fn async_embed_awaitable<'py>(
&self,
py: Python<'py>,
prompt: &str,
metadata: &Option<Value>,
) -> PyResult<Bound<'py, PyAny>> {
let metadata = to_py(py, metadata)?;
self.0
.bind(py)
.call_method1("_get_async_embedding", (prompt, metadata))
}
fn metadata_kwargs<'py>(
py: Python<'py>,
metadata: Option<&Value>,
@ -62,7 +49,8 @@ impl PythonEmbedder {
Ok(kwargs)
}
pub(super) fn async_embedding_coroutine(
/// The awaitable of `_get_async_embedding(prompt, metadata=...)`, to run in the caller's loop.
pub(super) fn async_embedding(
&self,
py: Python<'_>,
prompt: &str,
@ -82,35 +70,8 @@ impl PythonEmbedder {
.map(|value| value as f32)
.collect())
}
}
impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder {
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
let result = Python::attach(|py| -> PyResult<Vec<f64>> {
let metadata = to_py(py, &metadata)?;
self.0
.bind(py)
.call_method1("_get_embedding", (prompt, metadata))?
.extract()
})
.map_err(|_| Error::Unavailable)?;
Ok(result.into_iter().map(|value| value as f32).collect())
}
fn async_embed(
&self,
_prompt: &str,
_metadata: Option<&Value>,
) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
let seeded = PREPARED_EMBEDDING
.try_with(Clone::clone)
.unwrap_or(Err(Error::Unavailable));
std::future::ready(seeded)
}
}
impl litellm_cache_redis_semantic::Embedder for PythonEmbedder {
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
fn embed_sync(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
Python::attach(|py| {
let kwargs = Self::metadata_kwargs(py, metadata)?;
Self::extract(self.0.bind(py).call_method(
@ -122,15 +83,38 @@ impl litellm_cache_redis_semantic::Embedder for PythonEmbedder {
.map_err(|_| Error::Unavailable)
}
fn seeded_embedding() -> Result<Vec<f32>, Error> {
PREPARED_EMBEDDING
.try_with(Clone::clone)
.unwrap_or(Err(Error::Unavailable))
}
}
impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder {
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
self.embed_sync(prompt, metadata)
}
fn async_embed(
&self,
_prompt: &str,
_metadata: Option<&Value>,
) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
let seeded = PREPARED_EMBEDDING
.try_with(Clone::clone)
.unwrap_or(Err(Error::Unavailable));
std::future::ready(seeded)
std::future::ready(Self::seeded_embedding())
}
}
impl litellm_cache_redis_semantic::Embedder for PythonEmbedder {
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
self.embed_sync(prompt, metadata)
}
fn async_embed(
&self,
_prompt: &str,
_metadata: Option<&Value>,
) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
std::future::ready(Self::seeded_embedding())
}
}
@ -152,5 +136,9 @@ mod tests {
let unscoped =
litellm_cache_redis_semantic::Embedder::async_embed(&embedder, "prompt", None).await;
assert_eq!(unscoped, Err(Error::Unavailable));
let valkey = with_prepared_embedding(Ok(vec![0.5]), async move {
litellm_cache_valkey_semantic::Embedder::async_embed(&embedder, "prompt", None).await
});
assert_eq!(valkey.await, Ok(vec![0.5]));
}
}

View file

@ -11,6 +11,7 @@ use serde_json::Value;
use super::{
config::{CacheConfigProjection, NativeCacheConfig},
handle::CacheTestHandle,
identity::BackendIdentity,
native::NativeResponseCache,
};
@ -352,47 +353,41 @@ impl FacadeGuard {
facade: &Bound<'_, PyAny>,
service: &NativeResponseCache,
) -> PyResult<Self> {
let kind = service.kind();
let identity = service.identity();
let kind = identity.kind();
let cache_type = py.import("litellm.caching.caching")?.getattr("Cache")?;
if !facade.get_type().is(&cache_type) {
return Err(PyTypeError::new_err(
"only exact built-in Cache facades can be registered",
));
}
let cluster = matches!(service.topology(), Some(RedisTopology::Cluster { .. }));
let (module, name, cache_kind) = match (kind, cluster) {
("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
("redis", false) => ("litellm.caching.redis_cache", "RedisCache", "redis"),
("redis_semantic", _) => (
"litellm.caching.redis_semantic_cache",
"RedisSemanticCache",
"redis-semantic",
),
let cluster = matches!(
identity,
BackendIdentity::Redis {
topology: RedisTopology::Cluster { .. },
..
}
);
let (module, name) = match (kind, cluster) {
("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache"),
("redis", false) => ("litellm.caching.redis_cache", "RedisCache"),
("redis", true) => ("litellm.caching.redis_cluster_cache", "RedisClusterCache"),
("redis_semantic", _) => ("litellm.caching.redis_semantic_cache", "RedisSemanticCache"),
("qdrant_semantic", _) => (
"litellm.caching.qdrant_semantic_cache",
"QdrantSemanticCache",
"qdrant-semantic",
),
("redis", true) => (
"litellm.caching.redis_cluster_cache",
"RedisClusterCache",
"redis",
),
("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache", "gcs"),
("valkey-semantic", false) => (
("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache"),
("valkey-semantic", _) => (
"litellm.caching.valkey_semantic_cache",
"ValkeySemanticCache",
"valkey-semantic",
),
("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"),
("azure-blob", _) => (
"litellm.caching.azure_blob_cache",
"AzureBlobCache",
"azure-blob",
),
("s3", _) => ("litellm.caching.s3_cache", "S3Cache", "s3"),
("disk", _) => ("litellm.caching.disk_cache", "DiskCache"),
("azure-blob", _) => ("litellm.caching.azure_blob_cache", "AzureBlobCache"),
("s3", _) => ("litellm.caching.s3_cache", "S3Cache"),
_ => unreachable!(),
};
let cache_kind = identity.cache_type();
let backend = facade.getattr("cache")?;
if facade.getattr("type")?.extract::<String>()? != cache_kind
|| !backend.get_type().is(&py.import(module)?.getattr(name)?)

View file

@ -261,7 +261,7 @@ impl CacheTestHandle {
index_name: String,
embedder: &Bound<'_, PyAny>,
) -> PyResult<Self> {
let python_embedder = PythonEmbedder::from_backend(embedder)?;
let python_embedder = PythonEmbedder::new(embedder.clone().unbind());
let service = NativeResponseCache::valkey_semantic(
&url,
similarity_threshold,

View file

@ -0,0 +1,511 @@
use std::{path::PathBuf, time::Duration};
use litellm_cache_redis::RedisTopology;
/// What makes a native backend the one a Python facade describes: the configuration a user can
/// observe on the Python object, captured once so facade projection and native construction
/// compare plain data instead of reaching into each backend type.
#[derive(Clone, Debug, PartialEq)]
pub(super) enum BackendIdentity {
Memory {
capacity: usize,
max_entry_bytes: Option<usize>,
default_ttl: Option<Duration>,
},
Redis {
topology: RedisTopology,
namespace: Option<String>,
default_ttl: Option<Duration>,
},
S3 {
bucket: String,
key_prefix: String,
region: String,
endpoint: Option<String>,
},
Gcs {
bucket_name: String,
key_prefix: String,
path_service_account: Option<String>,
},
Disk {
directory: PathBuf,
},
AzureBlob {
account_url: String,
container: String,
},
RedisSemantic {
index_name: String,
/// The backend stores the threshold as `f32`; a facade's `f64` is compared at that width.
similarity_threshold: f32,
},
ValkeySemantic {
index_name: String,
similarity_threshold: f64,
},
QdrantSemantic {
collection_name: String,
similarity_threshold: f64,
vector_size: u64,
embedding_model: String,
},
}
const TYPES: &str = "facade and native backend types must match";
impl BackendIdentity {
/// The native backend name reported to Python through `_CacheTestHandle.backend`.
pub(super) fn kind(&self) -> &'static str {
match self {
Self::Memory { .. } => "memory",
Self::Redis { .. } => "redis",
Self::S3 { .. } => "s3",
Self::Gcs { .. } => "gcs",
Self::ValkeySemantic { .. } => "valkey-semantic",
Self::RedisSemantic { .. } => "redis_semantic",
Self::QdrantSemantic { .. } => "qdrant_semantic",
Self::Disk { .. } => "disk",
Self::AzureBlob { .. } => "azure-blob",
}
}
/// The `LiteLLMCacheType` value a facade of this backend carries in `Cache.type`.
pub(super) fn cache_type(&self) -> &'static str {
match self {
Self::Memory { .. } => "local",
Self::Redis { .. } => "redis",
Self::S3 { .. } => "s3",
Self::Gcs { .. } => "gcs",
Self::ValkeySemantic { .. } => "valkey-semantic",
Self::RedisSemantic { .. } => "redis-semantic",
Self::QdrantSemantic { .. } => "qdrant-semantic",
Self::Disk { .. } => "disk",
Self::AzureBlob { .. } => "azure-blob",
}
}
/// The first difference between the facade's configuration (`self`) and the native
/// backend (`native`), in the order Python users see the attributes.
pub(super) fn mismatch(&self, native: &Self) -> Option<&'static str> {
let mut differences: Vec<(bool, &'static str)> = Vec::new();
let mut differs = |condition: bool, message: &'static str| {
differences.push((condition, message));
};
match (self, native) {
(
Self::Memory {
capacity,
max_entry_bytes,
default_ttl,
},
Self::Memory {
capacity: native_capacity,
max_entry_bytes: native_max_entry_bytes,
default_ttl: native_default_ttl,
},
) => {
differs(
default_ttl != native_default_ttl,
"facade and native backend default TTLs must match",
);
differs(
capacity != native_capacity,
"facade and native backend capacities must match",
);
differs(
max_entry_bytes != native_max_entry_bytes,
"facade and native backend item limits must match",
);
}
(
Self::Redis {
topology,
namespace,
default_ttl,
},
Self::Redis {
topology: native_topology,
namespace: native_namespace,
default_ttl: native_default_ttl,
},
) => {
differs(
default_ttl != native_default_ttl,
"facade and native backend default TTLs must match",
);
differs(
topology != native_topology,
"facade and native backend topologies must match",
);
differs(
namespace != native_namespace,
"facade and native backend namespaces must match",
);
}
(
Self::S3 {
bucket,
key_prefix,
region,
endpoint,
},
Self::S3 {
bucket: native_bucket,
key_prefix: native_key_prefix,
region: native_region,
endpoint: native_endpoint,
},
) => {
differs(
bucket != native_bucket,
"facade and native backend buckets must match",
);
differs(
key_prefix != native_key_prefix,
"facade and native backend key prefixes must match",
);
differs(
region != native_region,
"facade and native backend regions must match",
);
differs(
endpoint != native_endpoint,
"facade and native backend endpoints must match",
);
}
(
Self::Gcs {
bucket_name,
key_prefix,
path_service_account,
},
Self::Gcs {
bucket_name: native_bucket_name,
key_prefix: native_key_prefix,
path_service_account: native_path_service_account,
},
) => {
differs(
bucket_name != native_bucket_name,
"facade and native backend buckets must match",
);
differs(
key_prefix != native_key_prefix,
"facade and native backend key prefixes must match",
);
differs(
path_service_account != native_path_service_account,
"facade and native backend credentials must match",
);
}
(
Self::Disk { directory },
Self::Disk {
directory: native_directory,
},
) => {
let canonical = |path: &PathBuf| std::fs::canonicalize(path).ok();
differs(
canonical(directory) != canonical(native_directory),
"facade and native backend directories must match",
);
}
(
Self::AzureBlob {
account_url,
container,
},
Self::AzureBlob {
account_url: native_account_url,
container: native_container,
},
) => {
differs(
account_url != native_account_url || container != native_container,
"facade and native backend containers must match",
);
}
(
Self::RedisSemantic {
index_name,
similarity_threshold,
},
Self::RedisSemantic {
index_name: native_index_name,
similarity_threshold: native_similarity_threshold,
},
) => {
differs(
index_name != native_index_name,
"facade and native backend index names must match",
);
differs(
similarity_threshold != native_similarity_threshold,
"facade and native backend similarity thresholds must match",
);
}
(
Self::ValkeySemantic {
index_name,
similarity_threshold,
},
Self::ValkeySemantic {
index_name: native_index_name,
similarity_threshold: native_similarity_threshold,
},
) => {
differs(
index_name != native_index_name
|| similarity_threshold != native_similarity_threshold,
"facade and native semantic settings must match",
);
}
(
Self::QdrantSemantic {
collection_name,
similarity_threshold,
vector_size,
embedding_model,
},
Self::QdrantSemantic {
collection_name: native_collection_name,
similarity_threshold: native_similarity_threshold,
vector_size: native_vector_size,
embedding_model: native_embedding_model,
},
) => {
differs(
collection_name != native_collection_name,
"facade and native backend collections must match",
);
differs(
similarity_threshold != native_similarity_threshold,
"facade and native backend similarity thresholds must match",
);
differs(
vector_size != native_vector_size,
"facade and native backend vector sizes must match",
);
differs(
embedding_model != native_embedding_model,
"facade and native backend embedding models must match",
);
}
_ => return Some(TYPES),
}
differences
.into_iter()
.find_map(|(condition, message)| condition.then_some(message))
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use litellm_cache_redis::{RedisNode, RedisTopology};
use super::BackendIdentity;
fn memory() -> BackendIdentity {
BackendIdentity::Memory {
capacity: 200,
max_entry_bytes: Some(1024),
default_ttl: Some(Duration::from_secs(60)),
}
}
fn redis() -> BackendIdentity {
BackendIdentity::Redis {
topology: RedisTopology::Standalone,
namespace: Some("team".into()),
default_ttl: Some(Duration::from_secs(60)),
}
}
fn s3() -> BackendIdentity {
BackendIdentity::S3 {
bucket: "bucket".into(),
key_prefix: "cache/".into(),
region: "us-east-1".into(),
endpoint: None,
}
}
fn gcs() -> BackendIdentity {
BackendIdentity::Gcs {
bucket_name: "bucket".into(),
key_prefix: "cache/".into(),
path_service_account: Some("credentials.json".into()),
}
}
fn azure() -> BackendIdentity {
BackendIdentity::AzureBlob {
account_url: "https://account.blob.core.windows.net".into(),
container: "cache".into(),
}
}
fn redis_semantic() -> BackendIdentity {
BackendIdentity::RedisSemantic {
index_name: "idx".into(),
similarity_threshold: 0.8,
}
}
#[test]
fn redis_semantic_thresholds_compare_at_backend_precision() {
let facade = BackendIdentity::RedisSemantic {
index_name: "idx".into(),
similarity_threshold: 0.8_f64 as f32,
};
assert_eq!(facade.mismatch(&redis_semantic()), None);
}
fn valkey_semantic() -> BackendIdentity {
BackendIdentity::ValkeySemantic {
index_name: "idx".into(),
similarity_threshold: 0.8,
}
}
fn qdrant() -> BackendIdentity {
BackendIdentity::QdrantSemantic {
collection_name: "collection".into(),
similarity_threshold: 0.8,
vector_size: 1536,
embedding_model: "text-embedding-3-small".into(),
}
}
#[test]
fn identical_identities_have_no_mismatch() {
for identity in [
memory(),
redis(),
s3(),
gcs(),
azure(),
redis_semantic(),
valkey_semantic(),
qdrant(),
BackendIdentity::Disk {
directory: std::env::temp_dir(),
},
] {
assert_eq!(identity.mismatch(&identity), None, "{identity:?}");
}
}
#[test]
fn different_kinds_report_a_type_mismatch() {
assert_eq!(
memory().mismatch(&redis()),
Some("facade and native backend types must match")
);
assert_eq!(
redis_semantic().mismatch(&valkey_semantic()),
Some("facade and native backend types must match")
);
}
#[test]
fn the_first_differing_field_names_the_mismatch() {
let BackendIdentity::Memory { capacity, .. } = memory() else {
unreachable!()
};
assert_eq!(
memory().mismatch(&BackendIdentity::Memory {
capacity: capacity + 1,
max_entry_bytes: Some(1),
default_ttl: Some(Duration::from_secs(60)),
}),
Some("facade and native backend capacities must match")
);
assert_eq!(
memory().mismatch(&BackendIdentity::Memory {
capacity,
max_entry_bytes: Some(1),
default_ttl: Some(Duration::from_secs(61)),
}),
Some("facade and native backend default TTLs must match")
);
assert_eq!(
redis().mismatch(&BackendIdentity::Redis {
topology: RedisTopology::Cluster {
startup_nodes: vec![RedisNode {
host: "node".into(),
port: 7000,
}],
},
namespace: None,
default_ttl: Some(Duration::from_secs(60)),
}),
Some("facade and native backend topologies must match")
);
assert_eq!(
s3().mismatch(&BackendIdentity::S3 {
bucket: "bucket".into(),
key_prefix: "cache/".into(),
region: "us-east-1".into(),
endpoint: Some("http://localhost:9000".into()),
}),
Some("facade and native backend endpoints must match")
);
assert_eq!(
gcs().mismatch(&BackendIdentity::Gcs {
bucket_name: "bucket".into(),
key_prefix: "cache/".into(),
path_service_account: None,
}),
Some("facade and native backend credentials must match")
);
assert_eq!(
azure().mismatch(&BackendIdentity::AzureBlob {
account_url: "https://account.blob.core.windows.net".into(),
container: "other".into(),
}),
Some("facade and native backend containers must match")
);
assert_eq!(
valkey_semantic().mismatch(&BackendIdentity::ValkeySemantic {
index_name: "idx".into(),
similarity_threshold: 0.9,
}),
Some("facade and native semantic settings must match")
);
assert_eq!(
qdrant().mismatch(&BackendIdentity::QdrantSemantic {
collection_name: "collection".into(),
similarity_threshold: 0.8,
vector_size: 1536,
embedding_model: "text-embedding-3-large".into(),
}),
Some("facade and native backend embedding models must match")
);
}
#[test]
fn disk_directories_compare_canonically() {
let directory = std::env::temp_dir();
let mut indirect = directory.clone();
indirect.push(".");
assert_eq!(
BackendIdentity::Disk {
directory: directory.clone()
}
.mismatch(&BackendIdentity::Disk {
directory: indirect
}),
None
);
assert_eq!(
BackendIdentity::Disk { directory }.mismatch(&BackendIdentity::Disk {
directory: "/definitely/missing".into()
}),
Some("facade and native backend directories must match")
);
}
}

View file

@ -5,11 +5,11 @@ mod embedder;
mod facade;
mod future;
mod handle;
mod identity;
mod native;
mod request;
mod resolver;
mod semantic;
mod semantic_step;
use litellm_cache::Error;
use pyo3::{

File diff suppressed because it is too large Load diff

View file

@ -1,7 +1,7 @@
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use litellm_cache::ExactCacheContext;
use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest};
use litellm_cache::{ExactCacheContext, SemanticCacheContext};
use litellm_cache_response::{CacheControls, CacheKeyField, CacheKeyInput, ResponseCacheRequest};
use litellm_host_python::from_py;
use pyo3::{exceptions::PyValueError, prelude::*};
use serde::Deserialize;
@ -36,6 +36,99 @@ pub(super) struct NativeRequest {
pub(super) scope: Option<String>,
}
impl NativeRequest {
pub(super) fn exact(&self) -> ResponseCacheRequest<ExactCacheContext> {
ResponseCacheRequest {
key: self.key.clone(),
controls: self.controls,
context: ExactCacheContext { ttl: self.ttl },
max_age: self.max_age,
}
}
/// The request as a semantic backend that keys on the caller's scope sees it.
pub(super) fn semantic(&self) -> ResponseCacheRequest<SemanticCacheContext> {
self.semantic_with(self.key.clone(), self.scope.clone())
}
/// The request keyed the way Python's Valkey semantic cache keys it: prompt fields drop out
/// and the tenant identifiers for `scope` join the key.
pub(super) fn scoped_semantic(
&self,
scope: &str,
) -> ResponseCacheRequest<SemanticCacheContext> {
self.semantic_with(semantic_key(self, scope), Some(scope.to_owned()))
}
fn semantic_with(
&self,
key: CacheKeyInput,
scope: Option<String>,
) -> ResponseCacheRequest<SemanticCacheContext> {
ResponseCacheRequest {
key,
controls: self.controls,
context: SemanticCacheContext {
input: self.input.clone(),
messages: self.messages.clone(),
metadata: self.metadata.clone(),
scope,
ttl: self.ttl,
},
max_age: self.max_age,
}
}
}
fn semantic_key(request: &NativeRequest, scope: &str) -> CacheKeyInput {
let mut key = request.key.clone();
if key.preset.is_some() {
return key;
}
key.fields
.retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input"));
const TENANT: [&str; 3] = [
"user_api_key",
"user_api_key_team_id",
"user_api_key_org_id",
];
let end_user = (scope == "end_user").then_some("user_api_key_end_user_id");
for name in TENANT.into_iter().chain(end_user) {
let sources = [
request.metadata.as_ref(),
request.litellm_metadata.as_ref(),
request
.litellm_params
.as_ref()
.and_then(|params| params.get("metadata")),
request
.litellm_params
.as_ref()
.and_then(|params| params.get("litellm_metadata")),
];
let Some(value) = sources.into_iter().flatten().find_map(|source| {
source
.as_object()
.and_then(|values| values.get(name))
.filter(|value| !value.is_null())
}) else {
continue;
};
let value = match value {
Value::Null => continue,
Value::String(text) => text.clone(),
other => other.to_string(),
};
key.fields.push(CacheKeyField {
name: name.to_owned(),
value: Some(value),
api_parameter: true,
internal_parameter: false,
});
}
key
}
pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<NativeRequest> {
let input: RequestInput = from_py(value)?;
request_input(input)
@ -76,3 +169,90 @@ pub(super) fn now() -> Duration {
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key};
use serde_json::json;
use sha2::{Digest, Sha256};
use super::*;
fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest {
NativeRequest {
key,
controls: CacheControls::default(),
ttl: None,
max_age: None,
messages: Some(json!([{"role": "user", "content": "prompt"}])),
input: None,
metadata: Some(metadata),
litellm_metadata: None,
litellm_params: None,
scope: None,
}
}
#[test]
fn semantic_key_matches_python_scope_material() {
let key = CacheKeyInput {
fields: vec![
CacheKeyField {
name: "model".to_owned(),
value: Some("gpt-4.1".to_owned()),
api_parameter: true,
internal_parameter: false,
},
CacheKeyField {
name: "messages".to_owned(),
value: Some("prompt".to_owned()),
api_parameter: true,
internal_parameter: false,
},
],
..Default::default()
};
let request = native_request(
key,
json!({"user_api_key": "k1", "user_api_key_team_id": null}),
);
let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1"));
assert_eq!(cache_key(&semantic_key(&request, "key")), expected);
assert_eq!(cache_key(&request.scoped_semantic("key").key), expected);
let end_user_request = native_request(
request.key.clone(),
json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}),
);
let expected = format!(
"{:x}",
Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1")
);
assert_eq!(
cache_key(&semantic_key(&end_user_request, "end_user")),
expected
);
let preset_request = native_request(
CacheKeyInput {
preset: Some("preset-key".to_owned()),
..Default::default()
},
json!({"user_api_key": "k1"}),
);
assert_eq!(
semantic_key(&preset_request, "end_user").preset.as_deref(),
Some("preset-key")
);
assert!(semantic_key(&preset_request, "end_user").fields.is_empty());
assert_eq!(preset_request.semantic().context.scope, None);
assert_eq!(
preset_request
.scoped_semantic("end_user")
.context
.scope
.as_deref(),
Some("end_user")
);
}
}

View file

@ -1,7 +1,6 @@
use std::collections::VecDeque;
use std::{collections::VecDeque, time::Duration};
use litellm_cache::Error;
use litellm_cache_redis_semantic::prompt_from_context;
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
use pyo3::{
PyTraverseError, PyVisit,
@ -23,29 +22,104 @@ pub(super) enum SemanticOperation {
StoreBatch(VecDeque<(NativeRequest, Value)>),
}
/// What an exception from the Python embedder means for the operation.
#[derive(Clone, Copy)]
pub(super) enum EmbeddingFailure {
/// Raise the Python exception unchanged.
Propagate,
/// Treat the embedding as unavailable and let the backend report that.
Unavailable,
}
enum Phase {
Start,
AwaitingEmbedding,
AwaitingBackend,
}
pub(super) struct SemanticBody {
/// Runs a semantic cache operation whose embedding comes from Python: await the Python
/// embedder in the caller's event loop, seed the native backend with the vector, await the
/// backend, and repeat for each entry of a batch.
pub(super) struct SemanticExecution {
service: NativeResponseCache,
embedder: PythonEmbedder,
failure: EmbeddingFailure,
operation: SemanticOperation,
pending: Option<(NativeRequest, Option<Value>)>,
phase: Phase,
now: Duration,
}
impl SemanticBody {
pub(super) fn new(service: NativeResponseCache, operation: SemanticOperation) -> Self {
impl SemanticExecution {
pub(super) fn new(
service: NativeResponseCache,
embedder: PythonEmbedder,
failure: EmbeddingFailure,
operation: SemanticOperation,
) -> Self {
Self {
service,
embedder,
failure,
operation,
pending: None,
phase: Phase::Start,
now: now(),
}
}
/// Takes the next entry of the operation; `None` once a batch is exhausted.
fn next_pending(&mut self) -> Option<(NativeRequest, Option<Value>)> {
match &mut self.operation {
SemanticOperation::Lookup(request) => Some((request.clone(), None)),
SemanticOperation::Store(request, response) => {
Some((request.clone(), Some(std::mem::take(response))))
}
SemanticOperation::StoreBatch(queue) => queue
.pop_front()
.map(|(request, response)| (request, Some(response))),
}
}
fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
let Some(pending) = self.next_pending() else {
return Ok(ExecutionStep::Return(py.None()));
};
let (request, response) = &pending;
let enabled = match response {
None => request.controls.reads(),
Some(_) => request.controls.writes(),
};
let input = enabled
.then(|| self.service.embedding_input(request))
.flatten();
self.pending = Some(pending);
let Some(input) = input else {
return self.backend_step(py, Err(Error::Unavailable));
};
let awaitable =
self.embedder
.async_embedding(py, &input.prompt, input.metadata.as_ref())?;
self.phase = Phase::AwaitingEmbedding;
Ok(ExecutionStep::Await(awaitable))
}
fn embedded(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<ExecutionStep> {
let seed = match result {
Ok(vector) => {
PythonEmbedder::extract(vector.into_bound(py)).map_err(|_| Error::Unavailable)
}
Err(error) => match self.failure {
EmbeddingFailure::Propagate => return Err(error),
EmbeddingFailure::Unavailable if error.is_instance_of::<PyException>(py) => {
Err(Error::Unavailable)
}
EmbeddingFailure::Unavailable => return Err(error),
},
};
self.backend_step(py, seed)
}
fn backend_step(
&mut self,
py: Python<'_>,
@ -56,11 +130,12 @@ impl SemanticBody {
PyRuntimeError::new_err("semantic execution resumed without a pending operation")
})?;
let service = self.service.clone();
let now = self.now;
let future = async move {
match response {
None => service.async_lookup(&request, now()).await,
None => service.async_lookup(&request, now).await,
Some(response) => service
.async_store(&request, response, now())
.async_store(&request, response, now)
.await
.map(|_| None),
}
@ -68,106 +143,45 @@ impl SemanticBody {
let awaitable = run_async(py, with_prepared_embedding(seed, future), cache_error)?;
Ok(ExecutionStep::Await(awaitable.unbind()))
}
fn resume_py(
&mut self,
py: Python<'_>,
result: Option<PyResult<Py<PyAny>>>,
) -> PyResult<ExecutionStep> {
match (&self.phase, result) {
(Phase::Start, None) => self.start(py),
(Phase::AwaitingEmbedding, Some(result)) => self.embedded(py, result),
(Phase::AwaitingBackend, Some(Err(error))) => Err(error),
(Phase::AwaitingBackend, Some(Ok(value))) => {
let more = matches!(
&self.operation,
SemanticOperation::StoreBatch(queue) if !queue.is_empty()
);
if more {
self.phase = Phase::Start;
return self.start(py);
}
Ok(ExecutionStep::Return(value))
}
_ => Err(PyRuntimeError::new_err(
"invalid semantic cache execution state",
)),
}
}
}
impl ExecutionBody for SemanticBody {
fn resume(&mut self, mut result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
Python::attach(|py| {
loop {
match self.phase {
Phase::Start => {
if result.is_some() {
return Err(PyRuntimeError::new_err(
"semantic execution received a result before starting",
));
}
if self.pending.is_none() {
match &mut self.operation {
SemanticOperation::Lookup(request) => {
self.pending = Some((request.clone(), None));
}
SemanticOperation::Store(request, response) => {
let response = std::mem::replace(response, Value::Null);
self.pending = Some((request.clone(), Some(response)));
}
SemanticOperation::StoreBatch(queue) => {
let Some((request, response)) = queue.pop_front() else {
return Ok(ExecutionStep::Return(py.None()));
};
self.pending = Some((request, Some(response)));
}
}
}
let (request, _) = self.pending.as_ref().ok_or_else(|| {
PyRuntimeError::new_err("semantic execution has no pending operation")
})?;
let semantic = NativeResponseCache::semantic_request(request);
let Some(prompt) = prompt_from_context(&semantic.context) else {
return self.backend_step(py, Err(Error::Unavailable));
};
let embedder = self.service.semantic_embedder().ok_or_else(|| {
PyRuntimeError::new_err(
"semantic execution requires a redis-semantic backend",
)
})?;
let coroutine = embedder.async_embedding_coroutine(
py,
&prompt,
semantic.context.metadata.as_ref(),
)?;
self.phase = Phase::AwaitingEmbedding;
return Ok(ExecutionStep::Await(coroutine));
}
Phase::AwaitingEmbedding => {
let result = result.take().ok_or_else(|| {
PyRuntimeError::new_err(
"semantic execution expected an embedding result",
)
})?;
let seed = match result {
Ok(value) => PythonEmbedder::extract(value.into_bound(py))
.map_err(|_| Error::Unavailable),
Err(error) => {
if !error.is_instance_of::<PyException>(py) {
return Err(error);
}
Err(Error::Unavailable)
}
};
return self.backend_step(py, seed);
}
Phase::AwaitingBackend => {
let result = result.take().ok_or_else(|| {
PyRuntimeError::new_err("semantic execution expected a backend result")
})?;
let value = match result {
Ok(value) => value,
Err(error) => return Err(error),
};
let more = matches!(
&self.operation,
SemanticOperation::StoreBatch(queue) if !queue.is_empty()
);
if more {
self.phase = Phase::Start;
continue;
}
return Ok(ExecutionStep::Return(value));
}
}
}
})
impl ExecutionBody for SemanticExecution {
fn resume(&mut self, result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
Python::attach(|py| self.resume_py(py, result))
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
if let Some(embedder) = self.service.semantic_embedder() {
embedder.traverse(visit)?;
}
Ok(())
self.embedder.traverse(visit)
}
}
pub(super) fn drive(py: Python<'_>, body: SemanticBody) -> PyResult<Bound<'_, PyAny>> {
pub(super) fn drive(py: Python<'_>, body: SemanticExecution) -> PyResult<Bound<'_, PyAny>> {
let execution = Py::new(py, Execution::new(body))?;
py.import("litellm.rust_bridge.lifecycle")?
.getattr("drive")?

View file

@ -1,249 +0,0 @@
use std::{sync::Arc, time::Duration};
use litellm_cache::SemanticCacheContext;
use litellm_cache_response::{ResponseCache, ResponseCacheCodec, ResponseCacheRequest};
use litellm_cache_valkey_semantic::{PreparedEmbedding, ValkeySemanticCache, prompt_from_context};
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
use serde_json::Value;
use super::{cache_error, embedder::PythonEmbedder};
pub(super) enum Op {
Lookup,
Store(Value),
StoreBatch(Vec<Value>),
}
#[derive(Clone, Copy)]
enum State {
Start,
AwaitingEmbedding,
AwaitingStorage,
Done,
}
pub(super) struct SemanticEmbedExecution {
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
embedder: PythonEmbedder,
requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
op: Op,
now: Option<Duration>,
prepared: Vec<Option<Vec<f32>>>,
index: usize,
state: State,
}
impl SemanticEmbedExecution {
pub(super) fn lookup(
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
embedder: PythonEmbedder,
request: ResponseCacheRequest<SemanticCacheContext>,
) -> Self {
Self {
backend,
embedder,
requests: vec![request],
op: Op::Lookup,
now: None,
prepared: vec![None],
index: 0,
state: State::Start,
}
}
pub(super) fn store(
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
embedder: PythonEmbedder,
request: ResponseCacheRequest<SemanticCacheContext>,
response: Value,
) -> Self {
Self {
backend,
embedder,
requests: vec![request],
op: Op::Store(response),
now: None,
prepared: vec![None],
index: 0,
state: State::Start,
}
}
pub(super) fn store_batch(
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
embedder: PythonEmbedder,
requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
responses: Vec<Value>,
) -> Self {
Self {
backend,
embedder,
prepared: vec![None; requests.len()],
requests,
op: Op::StoreBatch(responses),
now: None,
index: 0,
state: State::Start,
}
}
fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
if self.now.is_none() {
self.now = Some(super::request::now());
}
while self.index < self.requests.len() {
let request = &self.requests[self.index];
let enabled = match &self.op {
Op::Lookup => request.controls.reads(),
Op::Store(_) | Op::StoreBatch(_) => request.controls.writes(),
};
if !enabled {
self.index += 1;
continue;
}
let Some(prompt) = prompt_from_context(&request.context) else {
self.index += 1;
continue;
};
let metadata = request.context.metadata.clone();
let awaitable = self
.embedder
.async_embed_awaitable(py, &prompt, &metadata)?;
self.state = State::AwaitingEmbedding;
return Ok(ExecutionStep::Await(awaitable.unbind()));
}
self.state = State::AwaitingStorage;
self.storage_step(py)
}
fn storage_step(&self, py: Python<'_>) -> PyResult<ExecutionStep> {
let requests = self.requests.clone();
let prepared = self.prepared.clone();
let backend = Arc::clone(&self.backend);
let now = self
.now
.ok_or_else(|| PyRuntimeError::new_err("semantic cache timestamp is unavailable"))?;
let awaitable = match &self.op {
Op::Lookup => {
let Some(request) = requests.into_iter().next() else {
return Err(PyRuntimeError::new_err(
"semantic lookup requires one request",
));
};
match prepared.into_iter().next().flatten() {
Some(values) => {
let backend = backend.with_embedder(PreparedEmbedding(values));
let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
run_async(
py,
async move { cache.async_lookup(&request, now).await },
cache_error,
)?
}
None => {
let cache = Arc::new(ResponseCache::new(backend));
run_async(
py,
async move { cache.async_lookup(&request, now).await },
cache_error,
)?
}
}
}
Op::Store(response) => {
let Some(request) = requests.into_iter().next() else {
return Err(PyRuntimeError::new_err(
"semantic store requires one request",
));
};
let response = response.clone();
match prepared.into_iter().next().flatten() {
Some(values) => {
let backend = backend.with_embedder(PreparedEmbedding(values));
let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
run_async(
py,
async move { cache.async_store(&request, response, now).await },
cache_error,
)?
}
None => {
let cache = Arc::new(ResponseCache::new(backend));
run_async(
py,
async move { cache.async_store(&request, response, now).await },
cache_error,
)?
}
}
}
Op::StoreBatch(responses) => {
let responses = responses.clone();
run_async(
py,
async move {
for ((request, response), prepared) in
requests.into_iter().zip(responses).zip(prepared)
{
let Some(values) = prepared else {
continue;
};
let backend = backend.with_embedder(PreparedEmbedding(values));
let cache = ResponseCache::new(Arc::new(backend));
cache.async_store(&request, response, now).await?;
}
Ok(())
},
cache_error,
)?
}
};
Ok(ExecutionStep::Await(awaitable.unbind()))
}
fn resume_py(
&mut self,
py: Python<'_>,
result: Option<PyResult<Py<PyAny>>>,
) -> PyResult<ExecutionStep> {
match (self.state, result) {
(State::Start, None) => self.start(py),
(State::AwaitingEmbedding, Some(Ok(value))) => {
let values = value.bind(py).extract::<Vec<f64>>()?;
self.prepared[self.index] =
Some(values.into_iter().map(|value| value as f32).collect());
self.index += 1;
self.start(py)
}
(State::AwaitingStorage, Some(Ok(value))) => {
self.state = State::Done;
Ok(ExecutionStep::Return(value))
}
(_, Some(Err(error))) => Err(error),
_ => Err(PyRuntimeError::new_err(
"invalid semantic cache execution state",
)),
}
}
}
impl ExecutionBody for SemanticEmbedExecution {
fn resume(&mut self, result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
Python::attach(|py| self.resume_py(py, result))
}
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
self.embedder.traverse(visit)
}
}
pub(super) fn drive_semantic<'py>(
py: Python<'py>,
body: SemanticEmbedExecution,
) -> PyResult<Bound<'py, PyAny>> {
let execution = Py::new(py, Execution::new(body))?;
py.import("litellm.rust_bridge.lifecycle")?
.getattr("drive")?
.call1((execution,))
}

View file

@ -1,7 +1,4 @@
use std::collections::BTreeSet;
use litellm_core_utils::serde_compat::parse_str_bool;
use litellm_http::SslVerify;
use pyo3::{
exceptions::{PyAttributeError, PyRuntimeError, PyValueError},
prelude::*,
@ -33,36 +30,52 @@ impl From<ProjectionError> for PyErr {
}
}
pub(crate) struct Truthy(pub bool);
pub(crate) struct ExactTrue(pub bool);
pub(crate) struct StrBool(pub Option<bool>);
pub(crate) struct OptionalStrictString(pub Option<String>);
pub(crate) struct FalsyOptionalString(pub Option<String>);
pub(crate) struct TuningString(pub Option<String>);
pub(crate) struct StringCollection(pub Vec<String>);
pub(crate) struct SslVerifyInput(pub Option<SslVerify>);
pub(crate) struct FieldSpec<T> {
name: &'static str,
decode: fn(&Field<'_>) -> Result<T, ProjectionError>,
}
impl<T> FieldSpec<T> {
pub(crate) const fn new(
name: &'static str,
decode: fn(&Field<'_>) -> Result<T, ProjectionError>,
) -> Self {
Self { name, decode }
}
pub(crate) fn read(
&self,
snapshot: &Bound<'_, PyAny>,
group: &'static str,
) -> Result<T, ProjectionError> {
(self.decode)(&Field::read(snapshot, group, self.name)?)
}
}
pub(crate) struct Field<'py> {
path: &'static str,
group: &'static str,
name: &'static str,
value: Bound<'py, PyAny>,
}
impl<'py> Field<'py> {
pub(crate) fn new(path: &'static str, value: Bound<'py, PyAny>) -> Self {
Self { path, value }
pub(crate) fn new(group: &'static str, name: &'static str, value: Bound<'py, PyAny>) -> Self {
Self { group, name, value }
}
/// Reads `snapshot.<name>`, distinguishing a field the accessor never declared from a
/// descriptor that raised `AttributeError`.
pub(crate) fn read(
snapshot: &Bound<'py, PyAny>,
path: &'static str,
group: &'static str,
name: &'static str,
) -> Result<Self, ProjectionError> {
let name = path.rsplit('.').next().unwrap_or(path);
match snapshot.getattr(name) {
Ok(value) => Ok(Self::new(path, value)),
Ok(value) => Ok(Self::new(group, name, value)),
Err(error) if error.is_instance_of::<PyAttributeError>(snapshot.py()) => {
match Self::missing_field(snapshot, name) {
Ok(true) => Err(ProjectionError::InternalSchemaFailure(format!(
"{path}: missing snapshot field"
"{group}.{name}: missing snapshot field"
))),
_ => Err(error.into()),
}
@ -84,27 +97,49 @@ impl<'py> Field<'py> {
&& getter.is(object.getattr("__getattribute__")?))
}
fn expected(&self, expected: &'static str) -> Result<String, ProjectionError> {
pub(crate) fn path(&self) -> String {
format!("{}.{}", self.group, self.name)
}
/// A member of this field's collection, reported under the same path.
pub(crate) fn member(&self, value: Bound<'py, PyAny>) -> Self {
Self::new(self.group, self.name, value)
}
pub(crate) fn expected(&self, expected: &str) -> Result<String, ProjectionError> {
Ok(format!(
"{}: expected {expected}, got {}",
self.path,
self.path(),
self.value.get_type().name()?
))
}
fn invalid(&self, expected: &'static str) -> ProjectionError {
pub(crate) fn invalid(&self, expected: &str) -> ProjectionError {
match self.expected(expected) {
Ok(message) => ProjectionError::InvalidConfiguration(message),
Err(error) => error,
}
}
pub(crate) fn truthy(&self) -> Result<Truthy, ProjectionError> {
Ok(Truthy(self.value.is_truthy()?))
pub(crate) fn value(&self) -> &Bound<'py, PyAny> {
&self.value
}
pub(crate) fn exact_true(&self) -> ExactTrue {
ExactTrue(self.value.is(PyBool::new(self.value.py(), true)))
pub(crate) fn truthy(&self) -> Result<bool, ProjectionError> {
Ok(self.value.is_truthy()?)
}
pub(crate) fn exact_true(&self) -> bool {
self.value.is(PyBool::new(self.value.py(), true))
}
pub(crate) fn schema_bool(&self) -> Result<bool, ProjectionError> {
if !self.value.is_instance_of::<PyBool>() {
return Err(ProjectionError::InternalSchemaFailure(
self.expected("a Boolean")?,
));
}
Ok(self.exact_true())
}
pub(crate) fn strict_string(&self) -> Result<String, ProjectionError> {
@ -124,108 +159,366 @@ impl<'py> Field<'py> {
self.strict_string()
}
pub(crate) fn schema_bool(&self) -> Result<bool, ProjectionError> {
if !self.value.is_instance_of::<PyBool>() {
return Err(ProjectionError::InternalSchemaFailure(
self.expected("a Boolean")?,
));
}
Ok(self.exact_true().0)
}
pub(crate) fn str_bool(&self) -> Result<StrBool, ProjectionError> {
pub(crate) fn str_bool(&self) -> Result<Option<bool>, ProjectionError> {
if self.value.is_none() {
return Ok(StrBool(None));
return Ok(None);
}
Ok(StrBool(parse_str_bool(&self.strict_string()?)))
Ok(parse_str_bool(&self.strict_string()?))
}
pub(crate) fn optional_strict_string(&self) -> Result<OptionalStrictString, ProjectionError> {
pub(crate) fn optional_strict_string(&self) -> Result<Option<String>, ProjectionError> {
if self.value.is_none() {
return Ok(OptionalStrictString(None));
return Ok(None);
}
self.strict_string().map(Some).map(OptionalStrictString)
self.strict_string().map(Some)
}
pub(crate) fn falsy_optional_string(&self) -> Result<FalsyOptionalString, ProjectionError> {
if !self.truthy()?.0 {
return Ok(FalsyOptionalString(None));
pub(crate) fn falsy_optional_string(&self) -> Result<Option<String>, ProjectionError> {
if !self.truthy()? {
return Ok(None);
}
self.strict_string().map(Some).map(FalsyOptionalString)
self.strict_string().map(Some)
}
pub(crate) fn tuning_string(&self) -> Result<TuningString, ProjectionError> {
if !self.truthy()?.0 || !self.value.is_instance_of::<PyString>() {
return Ok(TuningString(None));
pub(crate) fn tuning_string(&self) -> Result<Option<String>, ProjectionError> {
if !self.truthy()? || !self.value.is_instance_of::<PyString>() {
return Ok(None);
}
self.strict_string().map(Some).map(TuningString)
self.strict_string().map(Some)
}
pub(crate) fn string_collection(&self) -> Result<StringCollection, ProjectionError> {
if !self.truthy()?.0 {
return Ok(StringCollection(Vec::new()));
pub(crate) fn string_collection(&self) -> Result<Vec<String>, ProjectionError> {
if !self.truthy()? {
return Ok(Vec::new());
}
if self.value.is_instance_of::<PyString>() {
return self
.strict_string()
.map(|value| StringCollection(vec![value]));
return self.strict_string().map(|value| vec![value]);
}
let values = self
.value
self.value
.try_iter()?
.filter_map(|item| {
let member = match item {
Ok(value) => Self::new(self.path, value),
Ok(value) => self.member(value),
Err(error) => return Some(Err(error.into())),
};
match member.truthy() {
Ok(Truthy(false)) => None,
Ok(Truthy(true)) => Some(member.strict_string()),
Ok(false) => None,
Ok(true) => Some(member.strict_string()),
Err(error) => Some(Err(error)),
}
})
.collect::<Result<Vec<_>, ProjectionError>>()?;
Ok(StringCollection(values))
.collect()
}
pub(crate) fn host_collection(&self) -> Result<StringCollection, ProjectionError> {
let values = self
.string_collection()?
.0
.into_iter()
.map(|host| litellm_http::media::normalize_host(&host))
.collect::<BTreeSet<_>>();
Ok(StringCollection(values.into_iter().collect()))
}
pub(crate) fn ssl_verify(&self) -> Result<SslVerifyInput, ProjectionError> {
pub(crate) fn optional_string_collection(
&self,
) -> Result<Option<Vec<String>>, ProjectionError> {
if self.value.is_none() {
return Ok(SslVerifyInput(None));
return Ok(None);
}
if self.value.is_instance_of::<PyBool>() {
return Ok(SslVerifyInput(Some(if self.exact_true().0 {
SslVerify::Enabled
} else {
SslVerify::Disabled
})));
}
if self.value.is_instance_of::<PyString>() {
let parsed = match self.str_bool()?.0 {
Some(true) => SslVerify::Enabled,
Some(false) => SslVerify::Disabled,
None => SslVerify::CaBundle(self.strict_string()?.into()),
};
return Ok(SslVerifyInput(Some(parsed)));
}
let context = self.value.py().import("ssl")?.getattr("SSLContext")?;
if self.value.is_instance(&context)? {
return Err(ProjectionError::UnsupportedLiveObject(self.expected(
"a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported",
)?));
}
Err(self.invalid("a Boolean, Boolean string, CA path, or None"))
self.string_collection().map(Some)
}
pub(crate) fn python_binding(&self) -> Option<Py<PyAny>> {
(!self.value.is_none()).then(|| self.value.clone().unbind())
}
}
#[cfg(test)]
mod tests;
mod tests {
use std::ffi::CString;
use pyo3::{
exceptions::{PyLookupError, PyRuntimeError, PyValueError},
types::PyDict,
};
use rstest::rstest;
use super::*;
fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
py.eval(&CString::new(source).unwrap(), None, None).unwrap()
}
#[rstest]
#[case("None", false, false)]
#[case("False", false, false)]
#[case("True", true, true)]
#[case("0", false, false)]
#[case("1", true, false)]
#[case("''", false, false)]
#[case("'false'", true, false)]
#[case("[]", false, false)]
#[case("[0]", true, false)]
#[case("{}", false, false)]
#[case("object()", true, false)]
fn boolean_operations_have_distinct_python_semantics(
#[case] source: &str,
#[case] truth: bool,
#[case] exact: bool,
) {
Python::initialize();
Python::attach(|py| {
let value = evaluate(py, source);
let field = Field::new("test", "flag", value.clone());
assert_eq!(field.truthy().unwrap(), truth);
assert_eq!(field.exact_true(), exact);
assert_eq!(
field.truthy().unwrap(),
py.import("builtins")
.unwrap()
.getattr("bool")
.unwrap()
.call1((value,))
.unwrap()
.extract::<bool>()
.unwrap()
);
});
}
#[rstest]
#[case("None", Ok(None), Ok(None), Ok(None))]
#[case("''", Ok(Some("")), Ok(None), Ok(None))]
#[case(
"' value '",
Ok(Some(" value ")),
Ok(Some(" value ")),
Ok(Some(" value "))
)]
#[case("[]", Err(()), Ok(None), Ok(None))]
#[case("0", Err(()), Ok(None), Ok(None))]
#[case("1", Err(()), Err(()), Ok(None))]
#[case("object()", Err(()), Err(()), Ok(None))]
fn string_operations_do_not_conflate_absence_and_type_checks(
#[case] source: &str,
#[case] strict: Result<Option<&str>, ()>,
#[case] fallback: Result<Option<&str>, ()>,
#[case] tuning: Result<Option<&str>, ()>,
) {
Python::initialize();
Python::attach(|py| {
let field = Field::new("test", "string", evaluate(py, source));
let owned =
|expected: Result<Option<&str>, ()>| expected.map(|value| value.map(str::to_owned));
assert_eq!(
field.optional_strict_string().map_err(|_| ()),
owned(strict)
);
assert_eq!(
field.falsy_optional_string().map_err(|_| ()),
owned(fallback)
);
assert_eq!(field.tuning_string().map_err(|_| ()), owned(tuning));
});
}
#[rstest]
#[case("None", None)]
#[case("' True '", Some(true))]
#[case("' fAlSe '", Some(false))]
#[case("'yes'", None)]
#[case("'1'", None)]
#[case("'unknown'", None)]
fn string_boolean_tokens_remain_separate_from_truthiness(
#[case] source: &str,
#[case] expected: Option<bool>,
) {
Python::initialize();
Python::attach(|py| {
assert_eq!(
Field::new("test", "flag", evaluate(py, source))
.str_bool()
.unwrap(),
expected
);
});
}
#[test]
fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
failure = LookupError('protocol failed')
cause = ValueError('cause')
context = RuntimeError('context')
def fail():
try:
raise context
except RuntimeError:
raise failure from cause
class Bool:
def __bool__(self): return fail()
class Length:
def __len__(self): return fail()
class Iter:
def __iter__(self): return fail()
class Next:
def __iter__(self): return self
def __next__(self): return fail()
class Descriptor:
@property
def flag(self): return fail()
values = (Bool(), Length(), Iter(), Next(), [Bool()])
descriptor = Descriptor()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let values = locals.get_item("values").unwrap().unwrap();
for value in values.try_iter().unwrap() {
let error = Field::new("test", "flag", value.unwrap())
.string_collection()
.err()
.unwrap();
let error = PyErr::from(error);
assert!(
error
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
assert!(error.is_instance_of::<PyLookupError>(py));
assert!(error.traceback(py).is_some());
assert!(
error
.value(py)
.getattr("__cause__")
.unwrap()
.is(locals.get_item("cause").unwrap().unwrap())
);
assert!(
error
.value(py)
.getattr("__context__")
.unwrap()
.is(locals.get_item("context").unwrap().unwrap())
);
}
let error = Field::read(
&locals.get_item("descriptor").unwrap().unwrap(),
"test",
"flag",
)
.err()
.unwrap();
assert!(
PyErr::from(error)
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
});
}
#[test]
fn identity_and_string_contents_do_not_invoke_unrelated_protocols() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
class Hostile:
def __bool__(self): raise AssertionError('bool called')
def __eq__(self, other): raise AssertionError('eq called')
def __str__(self): raise AssertionError('str called')
class Text(str):
def __str__(self): raise AssertionError('str called')
def strip(self): raise AssertionError('strip called')
def lower(self): raise AssertionError('lower called')
hostile = Hostile()
text = Text(' False ')
",
Some(&locals),
Some(&locals),
)
.unwrap();
let hostile = Field::new("test", "flag", locals.get_item("hostile").unwrap().unwrap());
assert!(!hostile.exact_true());
assert!(matches!(
hostile.strict_string(),
Err(ProjectionError::InvalidConfiguration(_))
));
let text = Field::new("test", "flag", locals.get_item("text").unwrap().unwrap());
assert_eq!(text.strict_string().unwrap(), " False ");
assert_eq!(text.str_bool().unwrap(), Some(false));
});
}
#[test]
fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
failure = AttributeError('descriptor failed')
class Snapshot:
@property
def flag(self): raise failure
snapshot = Snapshot()
class Dynamic:
def __getattr__(self, name): raise failure
class Intercepted:
def __getattribute__(self, name): raise failure
dynamic = Dynamic()
intercepted = Intercepted()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let snapshot = locals.get_item("snapshot").unwrap().unwrap();
let descriptor = PyErr::from(Field::read(&snapshot, "test", "flag").err().unwrap());
assert!(
descriptor
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
for name in ["dynamic", "intercepted"] {
let value = locals.get_item(name).unwrap().unwrap();
let error = PyErr::from(Field::read(&value, "test", "flag").err().unwrap());
assert!(
error
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
}
let missing = PyErr::from(Field::read(&snapshot, "test", "missing").err().unwrap());
assert!(missing.is_instance_of::<PyRuntimeError>(py));
assert!(missing.to_string().contains("test.missing"));
});
}
#[test]
fn configuration_errors_name_fields_without_exposing_values() {
Python::initialize();
Python::attach(|py| {
for source in [
"{'secret': 'do-not-print'}",
"['host.test', {'secret': 'do-not-print'}]",
] {
let field = Field::new("test", "setting", evaluate(py, source));
let error = PyErr::from(field.falsy_optional_string().err().unwrap());
assert!(error.is_instance_of::<PyValueError>(py));
assert!(error.to_string().contains("test.setting"));
assert!(!error.to_string().contains("do-not-print"));
}
let hosts = Field::new(
"url_policy",
"user_url_allowed_hosts",
evaluate(py, "['host.test', 1]"),
);
assert!(matches!(
hosts.string_collection(),
Err(ProjectionError::InvalidConfiguration(_))
));
assert!(matches!(
Field::new("test", "flag", evaluate(py, "1")).str_bool(),
Err(ProjectionError::InvalidConfiguration(_))
));
});
}
}

View file

@ -1,372 +0,0 @@
use std::ffi::CString;
use pyo3::{
exceptions::{PyLookupError, PyRuntimeError, PyValueError},
types::PyDict,
};
use rstest::rstest;
use super::*;
fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
py.eval(&CString::new(source).unwrap(), None, None).unwrap()
}
#[rstest]
#[case("None", false, false)]
#[case("False", false, false)]
#[case("True", true, true)]
#[case("0", false, false)]
#[case("1", true, false)]
#[case("''", false, false)]
#[case("'false'", true, false)]
#[case("[]", false, false)]
#[case("[0]", true, false)]
#[case("{}", false, false)]
#[case("object()", true, false)]
fn boolean_operations_have_distinct_python_semantics(
#[case] source: &str,
#[case] truth: bool,
#[case] exact: bool,
) {
Python::initialize();
Python::attach(|py| {
let value = evaluate(py, source);
let field = Field::new("test.flag", value.clone());
assert_eq!(field.truthy().unwrap().0, truth);
assert_eq!(field.exact_true().0, exact);
assert_eq!(
field.truthy().unwrap().0,
py.import("builtins")
.unwrap()
.getattr("bool")
.unwrap()
.call1((value,))
.unwrap()
.extract::<bool>()
.unwrap()
);
});
}
#[rstest]
#[case("None", Ok(None), Ok(None), Ok(None))]
#[case("''", Ok(Some("")), Ok(None), Ok(None))]
#[case(
"' value '",
Ok(Some(" value ")),
Ok(Some(" value ")),
Ok(Some(" value "))
)]
#[case("[]", Err(()), Ok(None), Ok(None))]
#[case("0", Err(()), Ok(None), Ok(None))]
#[case("1", Err(()), Err(()), Ok(None))]
#[case("object()", Err(()), Err(()), Ok(None))]
fn string_operations_do_not_conflate_absence_and_type_checks(
#[case] source: &str,
#[case] strict: Result<Option<&str>, ()>,
#[case] fallback: Result<Option<&str>, ()>,
#[case] tuning: Result<Option<&str>, ()>,
) {
Python::initialize();
Python::attach(|py| {
let field = Field::new("test.string", evaluate(py, source));
let owned =
|expected: Result<Option<&str>, ()>| expected.map(|value| value.map(str::to_owned));
assert_eq!(
field
.optional_strict_string()
.map(|value| value.0)
.map_err(|_| ()),
owned(strict)
);
assert_eq!(
field
.falsy_optional_string()
.map(|value| value.0)
.map_err(|_| ()),
owned(fallback)
);
assert_eq!(
field.tuning_string().map(|value| value.0).map_err(|_| ()),
owned(tuning)
);
});
}
#[rstest]
#[case("None", None)]
#[case("' True '", Some(true))]
#[case("' fAlSe '", Some(false))]
#[case("'yes'", None)]
#[case("'1'", None)]
#[case("'unknown'", None)]
fn string_boolean_tokens_remain_separate_from_truthiness(
#[case] source: &str,
#[case] expected: Option<bool>,
) {
Python::initialize();
Python::attach(|py| {
assert_eq!(
Field::new("test.flag", evaluate(py, source))
.str_bool()
.unwrap()
.0,
expected
);
});
}
#[rstest]
#[case("'EXAMPLE.TEST.'", vec!["example.test"])]
#[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])]
#[case("('B.test', 'a.test')", vec!["a.test", "b.test"])]
#[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])]
#[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])]
#[case("None", vec![])]
#[case("False", vec![])]
fn host_collection_is_owned_normalized_and_deterministic(
#[case] source: &str,
#[case] expected: Vec<&str>,
) {
Python::initialize();
Python::attach(|py| {
assert_eq!(
Field::new("url_policy.user_url_allowed_hosts", evaluate(py, source))
.host_collection()
.unwrap()
.0,
expected
);
});
}
#[test]
fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
failure = LookupError('protocol failed')
cause = ValueError('cause')
context = RuntimeError('context')
def fail():
try:
raise context
except RuntimeError:
raise failure from cause
class Bool:
def __bool__(self): return fail()
class Length:
def __len__(self): return fail()
class Iter:
def __iter__(self): return fail()
class Next:
def __iter__(self): return self
def __next__(self): return fail()
class Descriptor:
@property
def flag(self): return fail()
values = (Bool(), Length(), Iter(), Next(), [Bool()])
descriptor = Descriptor()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let values = locals.get_item("values").unwrap().unwrap();
for value in values.try_iter().unwrap() {
let error = Field::new("test.flag", value.unwrap())
.host_collection()
.err()
.unwrap();
let error = PyErr::from(error);
assert!(
error
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
assert!(error.is_instance_of::<PyLookupError>(py));
assert!(error.traceback(py).is_some());
assert!(
error
.value(py)
.getattr("__cause__")
.unwrap()
.is(locals.get_item("cause").unwrap().unwrap())
);
assert!(
error
.value(py)
.getattr("__context__")
.unwrap()
.is(locals.get_item("context").unwrap().unwrap())
);
}
let error = Field::read(
&locals.get_item("descriptor").unwrap().unwrap(),
"test.flag",
)
.err()
.unwrap();
assert!(
PyErr::from(error)
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
});
}
#[test]
fn identity_and_string_contents_do_not_invoke_unrelated_protocols() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
class Hostile:
def __bool__(self): raise AssertionError('bool called')
def __eq__(self, other): raise AssertionError('eq called')
def __str__(self): raise AssertionError('str called')
class Text(str):
def __str__(self): raise AssertionError('str called')
def strip(self): raise AssertionError('strip called')
def lower(self): raise AssertionError('lower called')
hostile = Hostile()
text = Text(' False ')
",
Some(&locals),
Some(&locals),
)
.unwrap();
let hostile = Field::new("test.flag", locals.get_item("hostile").unwrap().unwrap());
assert!(!hostile.exact_true().0);
assert!(matches!(
hostile.strict_string(),
Err(ProjectionError::InvalidConfiguration(_))
));
let text = Field::new("test.flag", locals.get_item("text").unwrap().unwrap());
assert_eq!(text.strict_string().unwrap(), " False ");
assert_eq!(text.str_bool().unwrap().0, Some(false));
});
}
#[test]
fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
failure = AttributeError('descriptor failed')
class Snapshot:
@property
def flag(self): raise failure
snapshot = Snapshot()
class Dynamic:
def __getattr__(self, name): raise failure
class Intercepted:
def __getattribute__(self, name): raise failure
dynamic = Dynamic()
intercepted = Intercepted()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let snapshot = locals.get_item("snapshot").unwrap().unwrap();
let descriptor = PyErr::from(Field::read(&snapshot, "test.flag").err().unwrap());
assert!(
descriptor
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
for name in ["dynamic", "intercepted"] {
let value = locals.get_item(name).unwrap().unwrap();
let error = PyErr::from(Field::read(&value, "test.flag").err().unwrap());
assert!(
error
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
}
let missing = PyErr::from(Field::read(&snapshot, "test.missing").err().unwrap());
assert!(missing.is_instance_of::<PyRuntimeError>(py));
assert!(missing.to_string().contains("test.missing"));
});
}
#[test]
fn configuration_errors_name_fields_without_exposing_values() {
Python::initialize();
Python::attach(|py| {
for source in [
"{'secret': 'do-not-print'}",
"['host.test', {'secret': 'do-not-print'}]",
] {
let field = Field::new("test.setting", evaluate(py, source));
let error = PyErr::from(field.falsy_optional_string().err().unwrap());
assert!(error.is_instance_of::<PyValueError>(py));
assert!(error.to_string().contains("test.setting"));
assert!(!error.to_string().contains("do-not-print"));
}
let hosts = Field::new(
"url_policy.user_url_allowed_hosts",
evaluate(py, "['host.test', 1]"),
);
assert!(matches!(
hosts.host_collection(),
Err(ProjectionError::InvalidConfiguration(_))
));
assert!(matches!(
Field::new("test.flag", evaluate(py, "1")).str_bool(),
Err(ProjectionError::InvalidConfiguration(_))
));
});
}
#[test]
fn projection_releases_the_source_collection() {
Python::initialize();
Python::attach(|py| {
let source = evaluate(py, "['A.test']");
let projected = Field::new("test.hosts", source.clone())
.host_collection()
.unwrap()
.0;
source.call_method1("append", ("b.test",)).unwrap();
assert_eq!(projected, ["a.test"]);
assert_eq!(
Field::new("test.hosts", source)
.host_collection()
.unwrap()
.0,
["a.test", "b.test"]
);
});
}
#[rstest]
#[case("True", Some(true))]
#[case("False", Some(false))]
#[case("1", None)]
#[case("None", None)]
#[case("[]", None)]
fn accessor_booleans_are_strict_schema_values(
#[case] source: &str,
#[case] expected: Option<bool>,
) {
Python::initialize();
Python::attach(|py| {
let result = Field::new("secret_manager.readable", evaluate(py, source)).schema_bool();
match expected {
Some(expected) => assert_eq!(result.unwrap(), expected),
None => {
let error = PyErr::from(result.unwrap_err());
assert!(error.is_instance_of::<PyRuntimeError>(py));
assert!(error.to_string().contains("secret_manager.readable"));
}
}
});
}

View file

@ -1,5 +1,5 @@
use std::{
collections::HashSet,
collections::{BTreeSet, HashSet},
path::{Path, PathBuf},
sync::{Arc, LazyLock, Mutex, PoisonError},
};
@ -10,9 +10,75 @@ use litellm_http::{
TlsSource, Unsupported,
media::{PublicDnsResolver, UrlPolicy},
};
use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
use pyo3::{
exceptions::PyValueError,
prelude::*,
types::{PyBool, PyDict, PyString},
};
use crate::{coercion::Field, python_settings::PythonSettings};
use crate::{
coercion::{Field, FieldSpec, ProjectionError},
python_settings::{PythonSettings, Snapshot},
};
const SSL_VERIFY: FieldSpec<Option<SslVerify>> = FieldSpec::new("ssl_verify", decode_ssl_verify);
const SSL_CERTIFICATE: FieldSpec<Option<String>> =
FieldSpec::new("ssl_certificate", |field| field.optional_strict_string());
const SSL_SECURITY_LEVEL: FieldSpec<Option<String>> =
FieldSpec::new("ssl_security_level", |field| field.tuning_string());
const SSL_ECDH_CURVE: FieldSpec<Option<String>> =
FieldSpec::new("ssl_ecdh_curve", |field| field.tuning_string());
const FORCE_IPV4: FieldSpec<bool> = FieldSpec::new("force_ipv4", |field| field.truthy());
const HTTP2: FieldSpec<bool> = FieldSpec::new("http2", |field| Ok(field.exact_true()));
const AIOHTTP_TRUST_ENV: FieldSpec<bool> =
FieldSpec::new("aiohttp_trust_env", |field| field.truthy());
const DISABLE_AIOHTTP_TRUST_ENV: FieldSpec<bool> =
FieldSpec::new("disable_aiohttp_trust_env", |field| field.truthy());
const DISABLE_AIOHTTP_TRANSPORT: FieldSpec<bool> =
FieldSpec::new("disable_aiohttp_transport", |field| Ok(field.exact_true()));
const USER_AGENT: FieldSpec<String> = FieldSpec::new("user_agent", |field| field.schema_string());
const USER_URL_VALIDATION: FieldSpec<bool> =
FieldSpec::new("user_url_validation", |field| field.truthy());
const USER_URL_ALLOWED_HOSTS: FieldSpec<Vec<String>> =
FieldSpec::new("user_url_allowed_hosts", decode_hosts);
fn decode_hosts(field: &Field<'_>) -> Result<Vec<String>, ProjectionError> {
Ok(field
.string_collection()?
.into_iter()
.map(|host| litellm_http::media::normalize_host(&host))
.collect::<BTreeSet<_>>()
.into_iter()
.collect())
}
fn decode_ssl_verify(field: &Field<'_>) -> Result<Option<SslVerify>, ProjectionError> {
let value = field.value();
if value.is_none() {
return Ok(None);
}
if value.is_instance_of::<PyBool>() {
return Ok(Some(if field.exact_true() {
SslVerify::Enabled
} else {
SslVerify::Disabled
}));
}
if value.is_instance_of::<PyString>() {
return Ok(Some(match field.str_bool()? {
Some(true) => SslVerify::Enabled,
Some(false) => SslVerify::Disabled,
None => SslVerify::CaBundle(field.strict_string()?.into()),
}));
}
let context = value.py().import("ssl")?.getattr("SSLContext")?;
if value.is_instance(&context)? {
return Err(ProjectionError::UnsupportedLiveObject(field.expected(
"a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported",
)?));
}
Err(field.invalid("a Boolean, Boolean string, CA path, or None"))
}
static POOL: LazyLock<HttpClientPool> =
LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver)));
@ -80,20 +146,20 @@ pub(crate) fn url_policy(py: Python<'_>) -> PyResult<UrlPolicy> {
project_url_policy(&PythonSettings::UrlPolicy.read(py)?)
}
fn project_url_policy(value: &Bound<'_, PyAny>) -> PyResult<UrlPolicy> {
fn project_url_policy(snapshot: &Snapshot<'_>) -> PyResult<UrlPolicy> {
Ok(UrlPolicy {
validate: Field::read(value, "url_policy.user_url_validation")?
.truthy()?
.0,
allowed_hosts: Field::read(value, "url_policy.user_url_allowed_hosts")?
.host_collection()?
.0,
validate: snapshot.read(&USER_URL_VALIDATION)?,
allowed_hosts: snapshot.read(&USER_URL_ALLOWED_HOSTS)?,
})
}
fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult<Option<SslVerify>> {
match kwargs.get_item("ssl_verify")? {
Some(value) => Ok(Field::new("request.ssl_verify", value).ssl_verify()?.0),
Some(value) => Ok(decode_ssl_verify(&Field::new(
"request",
"ssl_verify",
value,
))?),
None => Ok(None),
}
}
@ -106,39 +172,18 @@ fn for_call(call_ssl_verify: Option<SslVerify>, asynchronous: bool) -> HttpSetti
}
}
fn configured(value: &Bound<'_, PyAny>) -> PyResult<HttpSettingsLayer> {
fn configured(snapshot: &Snapshot<'_>) -> PyResult<HttpSettingsLayer> {
Ok(HttpSettingsLayer {
ssl_verify: Field::read(value, "http_settings.ssl_verify")?
.ssl_verify()?
.0,
ssl_certificate: Field::read(value, "http_settings.ssl_certificate")?
.optional_strict_string()?
.0
.map(PathBuf::from),
ssl_security_level: Field::read(value, "http_settings.ssl_security_level")?
.tuning_string()?
.0,
ssl_ecdh_curve: Field::read(value, "http_settings.ssl_ecdh_curve")?
.tuning_string()?
.0,
force_ipv4: Some(Field::read(value, "http_settings.force_ipv4")?.truthy()?.0),
http2: Some(Field::read(value, "http_settings.http2")?.exact_true().0),
aiohttp_trust_env: Some(
Field::read(value, "http_settings.aiohttp_trust_env")?
.truthy()?
.0,
),
disable_aiohttp_trust_env: Some(
Field::read(value, "http_settings.disable_aiohttp_trust_env")?
.truthy()?
.0,
),
disable_aiohttp_transport: Some(
Field::read(value, "http_settings.disable_aiohttp_transport")?
.exact_true()
.0,
),
user_agent: Some(Field::read(value, "http_settings.user_agent")?.schema_string()?),
ssl_verify: snapshot.read(&SSL_VERIFY)?,
ssl_certificate: snapshot.read(&SSL_CERTIFICATE)?.map(PathBuf::from),
ssl_security_level: snapshot.read(&SSL_SECURITY_LEVEL)?,
ssl_ecdh_curve: snapshot.read(&SSL_ECDH_CURVE)?,
force_ipv4: Some(snapshot.read(&FORCE_IPV4)?),
http2: Some(snapshot.read(&HTTP2)?),
aiohttp_trust_env: Some(snapshot.read(&AIOHTTP_TRUST_ENV)?),
disable_aiohttp_trust_env: Some(snapshot.read(&DISABLE_AIOHTTP_TRUST_ENV)?),
disable_aiohttp_transport: Some(snapshot.read(&DISABLE_AIOHTTP_TRANSPORT)?),
user_agent: Some(snapshot.read(&USER_AGENT)?),
..HttpSettingsLayer::default()
})
}
@ -150,12 +195,15 @@ mod tests {
use rstest::rstest;
use super::*;
use crate::python_settings::CONTRACT;
fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> {
fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
.unwrap()
}
fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Snapshot<'py> {
let source = format!(
"
import json
import types
defaults = dict(
ssl_verify=True,
@ -170,14 +218,13 @@ defaults = dict(
user_agent='litellm/test',
)
defaults.update(dict({overrides}))
settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads(contract)['http_settings']['fields']}})
settings = types.SimpleNamespace(**defaults)
"
);
let locals = PyDict::new(py);
locals.set_item("contract", CONTRACT).unwrap();
let source = std::ffi::CString::new(source).unwrap();
py.run(&source, Some(&locals), Some(&locals)).unwrap();
locals.get_item("settings").unwrap().unwrap()
PythonSettings::Http.snapshot(locals.get_item("settings").unwrap().unwrap())
}
#[test]
@ -395,7 +442,7 @@ user_agent='litellm/9.9.9',
Python::attach(|py| {
let value = py.eval(c"__import__('types').SimpleNamespace(user_url_validation=[], user_url_allowed_hosts=['B.test', 'a.test.', 'b.test'])", None, None).unwrap();
assert_eq!(
project_url_policy(&value).unwrap(),
project_url_policy(&PythonSettings::UrlPolicy.snapshot(value)).unwrap(),
UrlPolicy {
validate: false,
allowed_hosts: vec!["a.test".into(), "b.test".into()],
@ -419,4 +466,44 @@ user_agent='litellm/9.9.9',
let settings = HttpSettings::from_layers([for_call(None, asynchronous), opted_out]);
assert_eq!(settings.trust_proxy_env, expected);
}
#[rstest]
#[case("'EXAMPLE.TEST.'", vec!["example.test"])]
#[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])]
#[case("('B.test', 'a.test')", vec!["a.test", "b.test"])]
#[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])]
#[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])]
#[case("None", vec![])]
#[case("False", vec![])]
fn host_collection_is_owned_normalized_and_deterministic(
#[case] source: &str,
#[case] expected: Vec<&str>,
) {
Python::initialize();
Python::attach(|py| {
assert_eq!(
decode_hosts(&Field::new(
"url_policy",
"user_url_allowed_hosts",
evaluate(py, source)
))
.unwrap(),
expected
);
});
}
#[test]
fn projection_releases_the_source_collection() {
Python::initialize();
Python::attach(|py| {
let source = evaluate(py, "['A.test']");
let projected = decode_hosts(&Field::new("test", "hosts", source.clone())).unwrap();
source.call_method1("append", ("b.test",)).unwrap();
assert_eq!(projected, ["a.test"]);
assert_eq!(
decode_hosts(&Field::new("test", "hosts", source)).unwrap(),
["a.test", "b.test"]
);
});
}
}

View file

@ -7,7 +7,12 @@ mod http;
mod marshal;
mod python_settings;
mod routes;
mod token_counter;
#[allow(
dead_code,
reason = "secret-manager foundations await rollout activation"
)]
mod secrets;
mod tokenizer;
#[pymodule(gil_used = true)]
mod _native {
@ -32,7 +37,12 @@ mod _native {
#[pymodule_export]
use crate::routes::responses::ResponsesWebSocketConnection;
#[pymodule_export]
use crate::token_counter::TokenCounter;
use crate::routes::token_counter::TokenCounter;
#[cfg(feature = "huggingface")]
#[pymodule_export]
use crate::tokenizer::HuggingFaceEncoding;
#[pymodule_export]
use crate::tokenizer::Tokenizer;
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
@ -43,7 +53,7 @@ mod _native {
let dict = module.dict();
dict.set_item("_CacheTestHandle", py.get_type::<CacheTestHandle>())?;
dict.set_item("_CacheTestResolver", py.get_type::<CacheTestResolver>())?;
dict.set_item("_CacheTestBinding", py.get_type::<ResolvedCache>())
dict.set_item("_ResponseCacheRuntime", py.get_type::<ResolvedCache>())
}
}
@ -78,10 +88,13 @@ mod tests {
"achat_completions",
"ResponsesWebSocketConnection",
"TokenCounter",
"Tokenizer",
"gil_stats",
"process_state_started",
"reserve_process_for_forking",
];
#[cfg(feature = "huggingface")]
expected.push("HuggingFaceEncoding");
expected.sort_unstable();
let mut public_names: Vec<String> = native_module(py)

View file

@ -1,5 +1,7 @@
use pyo3::prelude::*;
use crate::coercion::{FieldSpec, ProjectionError};
const MODULE: &str = "litellm.rust_bridge.settings";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@ -8,28 +10,39 @@ pub(crate) enum PythonSettings {
UrlPolicy,
ProviderDefaults,
SecretManager,
SecretManagerBinding,
}
pub(crate) struct Snapshot<'py> {
group: PythonSettings,
value: Bound<'py, PyAny>,
}
impl Snapshot<'_> {
pub(crate) fn read<T>(&self, spec: &FieldSpec<T>) -> Result<T, ProjectionError> {
spec.read(&self.value, self.group.name())
}
}
impl PythonSettings {
#[cfg(test)]
pub(crate) const ALL: [Self; 4] = [
Self::Http,
Self::UrlPolicy,
Self::ProviderDefaults,
Self::SecretManager,
];
pub(crate) fn name(self) -> &'static str {
match self {
Self::Http => "http_settings",
Self::UrlPolicy => "url_policy",
Self::ProviderDefaults => "provider_defaults",
Self::SecretManager => "secret_manager",
Self::SecretManagerBinding => "secret_manager_binding",
}
}
pub(crate) fn read(self, py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
py.import(MODULE)?.getattr(self.name())?.call0()
pub(crate) fn read(self, py: Python<'_>) -> PyResult<Snapshot<'_>> {
let value = py.import(MODULE)?.getattr(self.name())?.call0()?;
Ok(Snapshot { group: self, value })
}
#[cfg(test)]
pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> {
Snapshot { group: self, value }
}
pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> {
@ -38,209 +51,98 @@ impl PythonSettings {
}
}
#[cfg(test)]
pub(crate) const CONTRACT: &str = include_str!("../python_settings.json");
#[cfg(test)]
mod tests {
use super::{CONTRACT, PythonSettings};
use pyo3::prelude::*;
use serde_json::{Value, json};
use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict};
struct SettingSpec {
group: &'static str,
name: &'static str,
adapter: &'static str,
precedence: &'static str,
sensitive: bool,
shapes: &'static [&'static str],
unsupported_live: Option<&'static str>,
}
const SETTINGS: &[SettingSpec] = &[
SettingSpec {
group: "http_settings",
name: "ssl_verify",
adapter: "SslVerifyInput",
precedence: "module_global",
sensitive: false,
shapes: &["none", "bool", "str"],
unsupported_live: Some("configuration_error"),
},
SettingSpec {
group: "http_settings",
name: "ssl_certificate",
adapter: "OptionalStrictString",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "ssl_security_level",
adapter: "TuningString",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "ssl_ecdh_curve",
adapter: "TuningString",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "force_ipv4",
adapter: "Truthy",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "http2",
adapter: "ExactTrue",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "aiohttp_trust_env",
adapter: "Truthy",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "disable_aiohttp_trust_env",
adapter: "Truthy",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "disable_aiohttp_transport",
adapter: "ExactTrue",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "http_settings",
name: "user_agent",
adapter: "StrictString",
precedence: "accessor",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "url_policy",
name: "user_url_validation",
adapter: "Truthy",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "url_policy",
name: "user_url_allowed_hosts",
adapter: "HostCollection",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "provider_defaults",
name: "vertex_project",
adapter: "FalsyOptionalString",
precedence: "module_global",
sensitive: true,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "provider_defaults",
name: "vertex_location",
adapter: "FalsyOptionalString",
precedence: "module_global",
sensitive: true,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "provider_defaults",
name: "enable_azure_ad_token_refresh",
adapter: "ExactTrue",
precedence: "module_global",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
SettingSpec {
group: "secret_manager",
name: "readable",
adapter: "StrictBool",
precedence: "accessor",
sensitive: false,
shapes: &[],
unsupported_live: None,
},
];
use super::PythonSettings;
use crate::coercion::FieldSpec;
#[test]
fn settings_manifest_matches_the_semantic_contract() {
pyo3::Python::initialize();
let manifest: Value = pyo3::Python::attach(|py| {
let value = py
.import("json")
.unwrap()
.call_method1("loads", (CONTRACT,))
.unwrap();
litellm_host_python::from_py(&value).unwrap()
fn declarations_select_the_decoder_and_read_only_the_requested_field() {
const TRUTHY: FieldSpec<bool> = FieldSpec::new("flag", |field| field.truthy());
const EXACT: FieldSpec<bool> = FieldSpec::new("flag", |field| Ok(field.exact_true()));
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
reads = []
class Settings:
value = 1
@property
def flag(self):
reads.append('flag')
return self.value
@property
def unrelated(self):
raise AssertionError('unrequested field')
settings = Settings()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let value = locals.get_item("settings").unwrap().unwrap();
let snapshot = PythonSettings::Http.snapshot(value.clone());
assert!(snapshot.read(&TRUTHY).unwrap());
assert!(!snapshot.read(&EXACT).unwrap());
value.setattr("value", true).unwrap();
assert!(snapshot.read(&EXACT).unwrap());
assert_eq!(
locals
.get_item("reads")
.unwrap()
.unwrap()
.extract::<Vec<String>>()
.unwrap(),
["flag", "flag", "flag"]
);
});
}
#[test]
fn declared_reads_preserve_descriptor_and_decoder_failures_and_name_missing_fields() {
const FLAG: FieldSpec<bool> = FieldSpec::new("flag", |field| field.truthy());
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
from types import SimpleNamespace
failure = AttributeError('read failed')
class Descriptor:
@property
def flag(self): raise failure
class Truth:
def __bool__(self): raise failure
values = (Descriptor(), SimpleNamespace(flag=Truth()))
",
Some(&locals),
Some(&locals),
)
.unwrap();
let failure = locals.get_item("failure").unwrap().unwrap();
for value in locals
.get_item("values")
.unwrap()
.unwrap()
.try_iter()
.unwrap()
{
let snapshot = PythonSettings::Http.snapshot(value.unwrap());
let error = PyErr::from(snapshot.read(&FLAG).unwrap_err());
assert!(error.value(py).is(&failure));
assert!(error.traceback(py).is_some());
}
let missing = PythonSettings::Http.snapshot(py.eval(c"object()", None, None).unwrap());
let error = PyErr::from(missing.read(&FLAG).unwrap_err());
assert!(error.is_instance_of::<PyRuntimeError>(py));
assert!(
error
.to_string()
.contains("http_settings.flag: missing snapshot field")
);
});
let expected: serde_json::Map<String, Value> = PythonSettings::ALL
.into_iter()
.map(|group| {
let fields: serde_json::Map<String, Value> = SETTINGS
.iter()
.filter(|spec| spec.group == group.name())
.map(|spec| {
(
spec.name.to_owned(),
json!({
"adapter": spec.adapter,
"required": true,
"precedence": spec.precedence,
"sensitive": spec.sensitive,
"shapes": spec.shapes,
"unsupported_live": spec.unsupported_live,
}),
)
})
.collect();
(
group.name().to_owned(),
json!({"version": 1, "fields": fields}),
)
})
.collect();
assert_eq!(manifest, Value::Object(expected));
}
}

View file

@ -3,6 +3,7 @@ pub(crate) mod chat_completions;
pub(crate) mod messages;
pub(crate) mod ocr;
pub(crate) mod responses;
pub(crate) mod token_counter;
#[cfg(test)]
mod tests {

View file

@ -130,6 +130,11 @@ impl RouteHost for OcrRouteHost {
}
fn classify(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::callback::python_error(py, source)
{
return Ok(original);
}
Ok(self.map_failure(py, ocr_error_to_pyerr(error)))
}

View file

@ -10,16 +10,33 @@ use litellm_auth_gcp::VertexAuth;
use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call};
use litellm_core::ocr::route::ocr_machine;
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_llms::base_llm::ocr::{
handler::OcrClient,
settings::{OcrSettings, Secrets},
use litellm_llms::base_llm::{
inference::secrets::{EnvironmentSecrets, SecretSource},
ocr::{handler::OcrClient, settings::OcrSettings},
};
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use crate::{coercion::Field, errors::RustBridgeDeclined, http, python_settings::PythonSettings};
use crate::{
coercion::FieldSpec,
errors::RustBridgeDeclined,
http,
python_settings::{PythonSettings, Snapshot},
};
const SECRET_MANAGER_READABLE: FieldSpec<bool> =
FieldSpec::new("readable", |field| field.schema_bool());
const VERTEX_PROJECT: FieldSpec<Option<String>> =
FieldSpec::new("vertex_project", |field| field.falsy_optional_string());
const VERTEX_LOCATION: FieldSpec<Option<String>> =
FieldSpec::new("vertex_location", |field| field.falsy_optional_string());
const ENABLE_AZURE_AD_TOKEN_REFRESH: FieldSpec<bool> =
FieldSpec::new("enable_azure_ad_token_refresh", |field| {
Ok(field.exact_true())
});
const SURFACE: LegacySurface = LegacySurface {
call_type: "ocr",
@ -62,33 +79,24 @@ fn run_ocr(
)
}
fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult<Secrets> {
if Field::read(secret_manager, "secret_manager.readable")?.schema_bool()? {
fn process_environment_secrets(snapshot: &Snapshot<'_>) -> PyResult<Arc<dyn SecretSource>> {
if snapshot.read(&SECRET_MANAGER_READABLE)? {
return Err(RustBridgeDeclined::new_err(
"a readable secret manager is configured and the Rust route only reads the process environment",
));
}
Ok(Arc::new(ProcessEnvironment))
Ok(Arc::new(EnvironmentSecrets))
}
fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
project_provider_defaults(&PythonSettings::ProviderDefaults.read(py)?)
}
fn project_provider_defaults(value: &Bound<'_, PyAny>) -> PyResult<OcrSettings> {
fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult<OcrSettings> {
Ok(OcrSettings {
vertex_project: Field::read(value, "provider_defaults.vertex_project")?
.falsy_optional_string()?
.0,
vertex_location: Field::read(value, "provider_defaults.vertex_location")?
.falsy_optional_string()?
.0,
enable_azure_ad_token_refresh: Field::read(
value,
"provider_defaults.enable_azure_ad_token_refresh",
)?
.exact_true()
.0,
vertex_project: snapshot.read(&VERTEX_PROJECT)?,
vertex_location: snapshot.read(&VERTEX_LOCATION)?,
enable_azure_ad_token_refresh: snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?,
..OcrSettings::from_environment(&ProcessEnvironment)
})
}
@ -120,6 +128,8 @@ mod tests {
use super::process_environment_secrets;
use crate::errors::RustBridgeDeclined;
use crate::python_settings::PythonSettings;
fn secret_manager<'py>(py: Python<'py>, readable: bool) -> Bound<'py, PyAny> {
let locals = PyDict::new(py);
locals.set_item("readable", readable).unwrap();
@ -132,12 +142,26 @@ mod tests {
locals.get_item("manager").unwrap().unwrap()
}
#[test]
fn a_readable_secret_manager_sends_the_call_back_to_python() {
Python::initialize();
Python::attach(|py| {
let declined = process_environment_secrets(
&PythonSettings::SecretManager.snapshot(secret_manager(py, true)),
)
.err()
.expect("the Rust route declines");
assert!(declined.is_instance_of::<RustBridgeDeclined>(py));
});
}
#[test]
fn provider_defaults_distinguish_falsey_values_and_exact_true() {
Python::initialize();
Python::attach(|py| {
let value = py.eval(c"__import__('types').SimpleNamespace(vertex_project=[], vertex_location=0, enable_azure_ad_token_refresh=1)", None, None).unwrap();
let projected = super::project_provider_defaults(&value).unwrap();
let snapshot = PythonSettings::ProviderDefaults.snapshot(value.clone());
let projected = super::project_provider_defaults(&snapshot).unwrap();
assert_eq!(projected.vertex_project, None);
assert_eq!(projected.vertex_location, None);
assert!(!projected.enable_azure_ad_token_refresh);
@ -146,12 +170,12 @@ mod tests {
value
.setattr("enable_azure_ad_token_refresh", true)
.unwrap();
let next = super::project_provider_defaults(&value).unwrap();
let next = super::project_provider_defaults(&snapshot).unwrap();
assert_eq!(next.vertex_project.as_deref(), Some("project"));
assert_eq!(next.vertex_location.as_deref(), Some("region"));
assert!(next.enable_azure_ad_token_refresh);
value.setattr("vertex_project", 1).unwrap();
let error = super::project_provider_defaults(&value).err().unwrap();
let error = super::project_provider_defaults(&snapshot).err().unwrap();
assert!(error.is_instance_of::<pyo3::exceptions::PyValueError>(py));
assert!(
error
@ -160,28 +184,4 @@ mod tests {
);
});
}
#[test]
fn a_readable_secret_manager_sends_the_call_back_to_python() {
Python::initialize();
Python::attach(|py| {
let declined = process_environment_secrets(&secret_manager(py, true))
.err()
.expect("the Rust route declines");
assert!(declined.is_instance_of::<RustBridgeDeclined>(py));
});
}
#[test]
fn without_a_readable_secret_manager_secrets_are_the_process_environment() {
Python::initialize();
Python::attach(|py| {
let secrets = process_environment_secrets(&secret_manager(py, false)).unwrap();
assert_eq!(
secrets.get("LITELLM_RUST_BRIDGE_UNSET_VARIABLE_FOR_TEST"),
None
);
assert_eq!(secrets.get("PATH"), std::env::var("PATH").ok());
});
}
}

View file

@ -0,0 +1,87 @@
use std::sync::Arc;
use std::{num::NonZero, thread::available_parallelism};
use litellm_host_python::{enter_native, run_async};
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
types::PyAny,
};
use tokio::sync::Semaphore;
use crate::errors::RustBridgeDeclined;
use crate::tokenizer::Tokenizer;
/// Counts the input tokens of a raw request body off the Python event loop with
/// the GIL released. Python owns which requests get here and what to do with
/// the count. At most one encode per core runs at a time; the rest wait in the
/// async task, where a cancelled Python awaiter drops them before any blocking
/// work is scheduled.
#[pyclass(frozen)]
pub(crate) struct TokenCounter {
inner: Arc<CoreTokenCounter>,
encode_slots: Arc<Semaphore>,
}
#[pymethods]
impl TokenCounter {
#[staticmethod]
#[pyo3(signature = (tokenizer, fast = false))]
fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult<Self> {
enter_native()?;
let inner = CoreTokenCounter::new(tokenizer.counter(py, fast));
Ok(Self {
inner: Arc::new(inner),
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
})
}
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
let counter = Arc::clone(&self.inner);
let encode_slots = Arc::clone(&self.encode_slots);
let body = body.to_vec();
run_async(
py,
async move {
let _slot = encode_slots
.acquire_owned()
.await
.map_err(|error| Error::Task(error.to_string()))?;
tokio::task::spawn_blocking(move || count_body(&counter, &body))
.await
.map_err(|error| Error::Task(error.to_string()))?
},
token_count_error_to_pyerr,
)
}
}
fn encode_parallelism() -> usize {
available_parallelism().map_or(1, NonZero::get)
}
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
let request = CountableRequest::parse(body)?;
counter.count_request(&request)
}
pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr {
let message = error.to_string();
match error {
Error::Load(_)
| Error::Ranks(_)
| Error::UnicodeClasses
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
Error::RequestParse(_)
| Error::MissingInput
| Error::FloatText
| Error::ContentBlock
| Error::ArrayItems
| Error::JsonSerialization(_)
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
}
}

View file

@ -0,0 +1,349 @@
use std::{fmt, future::Future, pin::Pin};
use litellm_core_utils::settings::Lookup;
use litellm_secrets::{
Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
};
use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler";
struct PythonSecretError(Py<PyBaseException>);
impl fmt::Debug for PythonSecretError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("PythonSecretError")
}
}
impl fmt::Display for PythonSecretError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("Python secret manager failed")
}
}
impl std::error::Error for PythonSecretError {}
pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option<PyErr> {
let Error::ExternalManager(source) = error else {
return None;
};
source
.downcast_ref::<PythonSecretError>()
.map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any()))
}
/// A secret manager whose reads execute in Python: a custom manager, a legacy compatible
/// client, or a manually assigned SDK client.
pub(crate) struct PythonSecretManager {
client: Py<PyAny>,
system: Option<KeyManagementSystem>,
/// The `key_manager` name Python's handler dispatches on.
key_manager: &'static str,
settings: Option<Py<PyAny>>,
}
impl PythonSecretManager {
pub(crate) fn new(
client: Py<PyAny>,
system: Option<KeyManagementSystem>,
settings: Option<Py<PyAny>>,
) -> Self {
Self {
client,
system,
key_manager: system.map_or("local", python_name),
settings,
}
}
fn read(&self, py: Python<'_>, name: &str) -> PyResult<Option<String>> {
let client = self.client.bind(py);
if self.system == Some(KeyManagementSystem::Custom)
|| (self.system.is_none() && client.hasattr("sync_read_secret")?)
{
let kwargs = PyDict::new(py);
kwargs.set_item("secret_name", name)?;
if self.system == Some(KeyManagementSystem::Custom) {
let optional_params = self
.settings
.as_ref()
.map(|settings| settings.bind(py).call_method0("model_dump"))
.transpose()?;
kwargs.set_item("optional_params", optional_params)?;
}
return client
.call_method("sync_read_secret", (), Some(&kwargs))?
.extract();
}
let kwargs = PyDict::new(py);
kwargs.set_item("client", client)?;
kwargs.set_item("key_manager", self.key_manager)?;
kwargs.set_item("secret_name", name)?;
kwargs.set_item(
"key_management_settings",
self.settings
.as_ref()
.map_or_else(|| py.None(), |settings| settings.clone_ref(py)),
)?;
py.import(HANDLER_MODULE)?
.getattr("get_secret_from_manager")?
.call((), Some(&kwargs))?
.extract()
}
}
/// The `KeyManagementSystem` value as Python spells it.
fn python_name(system: KeyManagementSystem) -> &'static str {
match system {
KeyManagementSystem::GoogleKms => "google_kms",
KeyManagementSystem::AzureKeyVault => "azure_key_vault",
KeyManagementSystem::AwsSecretManager => "aws_secret_manager",
KeyManagementSystem::GoogleSecretManager => "google_secret_manager",
KeyManagementSystem::HashicorpVault => "hashicorp_vault",
KeyManagementSystem::Cyberark => "cyberark",
KeyManagementSystem::Local => "local",
KeyManagementSystem::AwsKms => "aws_kms",
KeyManagementSystem::Custom => "custom",
}
}
impl ExternalSecretManager for PythonSecretManager {
fn system(&self) -> KeyManagementSystem {
self.system.unwrap_or(KeyManagementSystem::Custom)
}
fn read_secret<'a>(
&'a self,
name: &'a str,
_settings: &'a KeyManagementSettings,
_environment: &'a (dyn Lookup + Send + Sync),
) -> Pin<Box<dyn Future<Output = Result<Option<Secret>, Error>> + Send + 'a>> {
Box::pin(async move {
Python::attach(|py| {
self.read(py, name)
.map(|value| value.map(SecretValue::new).map(Secret::String))
.map_err(|error| {
Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py))))
})
})
})
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use litellm_secrets::{
FailurePolicy, KeyManagementSettings, KeyManagementSystem, OidcResolver, SecretManager,
SecretManagerState, SecretResolver,
};
use pyo3::{prelude::*, types::PyDict};
use super::{HANDLER_MODULE, PythonSecretManager, python_error, python_name};
#[tokio::test]
async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback() {
Python::initialize();
for failure_type in ["ValueError", "asyncio.CancelledError"] {
for fallback in [None, Some("environment-key")] {
let (reader, locals) = Python::attach(|py| {
let locals = PyDict::new(py);
locals.set_item("failure_type", failure_type).unwrap();
py.run(
c"
import asyncio
failure = eval(failure_type)('secret manager failed')
cause = RuntimeError('original cause')
context = RuntimeError('original context')
failure.__cause__ = cause
failure.__context__ = context
class Manager:
def sync_read_secret(self, secret_name):
raise failure
manager = Manager()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let reader = PythonSecretManager::new(
locals.get_item("manager").unwrap().unwrap().unbind(),
None,
None,
);
(reader, locals.unbind())
});
let resolver = SecretResolver::new(
Arc::new(SecretManagerState::new(
SecretManager::External(Arc::new(reader)),
KeyManagementSettings::default(),
)),
Arc::new(move |_: &str| fallback.map(str::to_owned)),
OidcResolver::default(),
)
.with_failure_policy(FailurePolicy::EnvironmentFallback);
let error = resolver.get_secret("API_KEY", None).await.unwrap_err();
Python::attach(|py| {
let original = python_error(py, &error).unwrap();
let locals = locals.bind(py);
assert!(
original
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] {
assert!(
original
.value(py)
.getattr(attribute)
.unwrap()
.is(locals.get_item(name).unwrap().unwrap())
);
}
assert!(original.traceback(py).is_some());
});
}
}
}
/// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and
/// removes the fake modules again.
fn with_fake_handler<'py>(py: Python<'py>, body: impl FnOnce(&Bound<'py, PyDict>)) {
let locals = PyDict::new(py);
py.run(
c"
import sys, types
calls = []
def get_secret_from_manager(**kwargs):
calls.append(kwargs)
return 'handled-' + kwargs['secret_name']
handler = types.ModuleType('litellm.secret_managers.secret_manager_handler')
handler.get_secret_from_manager = get_secret_from_manager
installed = {}
for name in ('litellm', 'litellm.secret_managers'):
if name not in sys.modules:
sys.modules[name] = types.ModuleType(name)
installed[name] = True
sys.modules['litellm.secret_managers.secret_manager_handler'] = handler
",
Some(&locals),
Some(&locals),
)
.unwrap();
body(&locals);
py.run(
c"
sys.modules.pop('litellm.secret_managers.secret_manager_handler', None)
for name in installed:
sys.modules.pop(name, None)
",
Some(&locals),
Some(&locals),
)
.unwrap();
}
#[test]
fn python_names_round_trip_through_serde() {
for system in [
KeyManagementSystem::GoogleKms,
KeyManagementSystem::AzureKeyVault,
KeyManagementSystem::AwsSecretManager,
KeyManagementSystem::GoogleSecretManager,
KeyManagementSystem::HashicorpVault,
KeyManagementSystem::Cyberark,
KeyManagementSystem::Local,
KeyManagementSystem::AwsKms,
KeyManagementSystem::Custom,
] {
assert_eq!(
serde_json::to_value(system).unwrap(),
serde_json::Value::String(python_name(system).to_owned())
);
}
}
#[test]
fn custom_readers_without_a_system_are_called_directly() {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
class Manager:
def __init__(self):
self.names = []
def sync_read_secret(self, secret_name, optional_params=None, timeout=None):
self.names.append(secret_name)
return 'direct-' + secret_name
manager = Manager()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let manager = locals.get_item("manager").unwrap().unwrap();
let reader = PythonSecretManager::new(manager.clone().unbind(), None, None);
assert_eq!(
reader.read(py, "API_KEY").unwrap().as_deref(),
Some("direct-API_KEY")
);
assert_eq!(
manager
.getattr("names")
.unwrap()
.extract::<Vec<String>>()
.unwrap(),
["API_KEY"]
);
});
}
#[test]
fn configured_systems_dispatch_through_the_python_handler_with_the_original_settings() {
Python::initialize();
Python::attach(|py| {
with_fake_handler(py, |locals| {
let client = py.eval(c"object()", None, None).unwrap();
let settings = py.eval(c"object()", None, None).unwrap();
let reader = PythonSecretManager::new(
client.clone().unbind(),
Some(KeyManagementSystem::AzureKeyVault),
Some(settings.clone().unbind()),
);
assert_eq!(
reader.read(py, "API_KEY").unwrap().as_deref(),
Some("handled-API_KEY")
);
assert!(py.import(HANDLER_MODULE).is_ok());
let calls = locals.get_item("calls").unwrap().unwrap();
let call = calls.get_item(0).unwrap().cast_into::<PyDict>().unwrap();
assert!(call.get_item("client").unwrap().unwrap().is(&client));
assert!(
call.get_item("key_management_settings")
.unwrap()
.unwrap()
.is(&settings)
);
assert_eq!(
call.get_item("key_manager")
.unwrap()
.unwrap()
.extract::<String>()
.unwrap(),
"azure_key_vault"
);
assert_eq!(
call.get_item("secret_name")
.unwrap()
.unwrap()
.extract::<String>()
.unwrap(),
"API_KEY"
);
});
});
}
}

View file

@ -0,0 +1,338 @@
use std::sync::Arc;
use litellm_secrets::{SecretManager, SecretManagerState};
use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem, SecretValue};
use pyo3::prelude::*;
use serde_json::Value;
use super::callback::PythonSecretManager;
use crate::{
coercion::{Field, FieldSpec, ProjectionError},
python_settings::{PythonSettings, Snapshot},
};
const SYSTEM: FieldSpec<Option<KeyManagementSystem>> =
FieldSpec::new("system", parse_optional_system);
const ACCESS_MODE: FieldSpec<AccessMode> = FieldSpec::new("access_mode", parse_access_mode);
const HOSTED_KEYS: FieldSpec<Option<Vec<String>>> =
FieldSpec::new("hosted_keys", |field| field.optional_string_collection());
const STORE_VIRTUAL_KEYS: FieldSpec<bool> =
FieldSpec::new("store_virtual_keys", |field| field.truthy());
const PREFIX_FOR_STORED_VIRTUAL_KEYS: FieldSpec<String> =
FieldSpec::new("prefix_for_stored_virtual_keys", |field| {
field.strict_string()
});
const PRIMARY_SECRET_NAME: FieldSpec<Option<String>> =
FieldSpec::new("primary_secret_name", |field| field.falsy_optional_string());
const KMS_KEY_ID: FieldSpec<Option<String>> =
FieldSpec::new("kms_key_id", |field| field.falsy_optional_string());
const CUSTOM_SECRET_MANAGER: FieldSpec<Option<String>> =
FieldSpec::new("custom_secret_manager", |field| {
field.falsy_optional_string()
});
const AWS_REGION_NAME: FieldSpec<Option<String>> =
FieldSpec::new("aws_region_name", |field| field.falsy_optional_string());
const AWS_ROLE_NAME: FieldSpec<Option<String>> =
FieldSpec::new("aws_role_name", |field| field.falsy_optional_string());
const AWS_SESSION_NAME: FieldSpec<Option<String>> =
FieldSpec::new("aws_session_name", |field| field.falsy_optional_string());
const AWS_EXTERNAL_ID: FieldSpec<Option<String>> =
FieldSpec::new("aws_external_id", |field| field.falsy_optional_string());
const AWS_PROFILE_NAME: FieldSpec<Option<String>> =
FieldSpec::new("aws_profile_name", |field| field.falsy_optional_string());
const AWS_WEB_IDENTITY_TOKEN: FieldSpec<Option<String>> =
FieldSpec::new("aws_web_identity_token", |field| {
field.falsy_optional_string()
});
const AWS_STS_ENDPOINT: FieldSpec<Option<String>> =
FieldSpec::new("aws_sts_endpoint", |field| field.falsy_optional_string());
const REPLICA_REGIONS: FieldSpec<Option<Vec<String>>> =
FieldSpec::new("replica_regions", |field| {
field.optional_string_collection()
});
const CLIENT: FieldSpec<Option<Py<PyAny>>> =
FieldSpec::new("client", |field| Ok(field.python_binding()));
const SETTINGS_OBJECT: FieldSpec<Option<Py<PyAny>>> =
FieldSpec::new("settings_object", |field| Ok(field.python_binding()));
/// `litellm.secret_manager_client` as the bridge classifies it.
#[derive(Debug)]
pub(crate) enum SecretManagerClient {
/// `None`: reads come from the process environment.
Local,
/// A custom manager, legacy compatible client, or manually assigned SDK client that keeps
/// executing in Python.
PythonCallback(Py<PyAny>),
}
/// One operation-local capture of the secret manager globals, taken while attached to Python.
#[derive(Debug)]
pub(crate) struct SecretManagerSnapshot {
pub(crate) client: SecretManagerClient,
pub(crate) system: Option<KeyManagementSystem>,
/// Typed settings that drive native routing: access mode and hosted keys.
pub(crate) settings: KeyManagementSettings,
/// The original `KeyManagementSettings` object, handed back to Python callbacks unchanged.
pub(crate) settings_object: Option<Py<PyAny>>,
}
impl SecretManagerSnapshot {
pub(crate) fn into_state(self) -> Arc<SecretManagerState> {
match self.client {
SecretManagerClient::Local => Arc::new(SecretManagerState::default()),
SecretManagerClient::PythonCallback(client) => Arc::new(SecretManagerState::new(
SecretManager::External(Arc::new(PythonSecretManager::new(
client,
self.system,
self.settings_object,
))),
self.settings,
)),
}
}
}
/// Reads and projects the secret manager settings group in one attached operation.
pub(crate) fn read(py: Python<'_>) -> PyResult<SecretManagerSnapshot> {
Ok(project(&PythonSettings::SecretManagerBinding.read(py)?)?)
}
pub(crate) fn project(snapshot: &Snapshot<'_>) -> Result<SecretManagerSnapshot, ProjectionError> {
let system = snapshot.read(&SYSTEM)?;
let access_mode = snapshot.read(&ACCESS_MODE)?;
let settings = KeyManagementSettings {
hosted_keys: snapshot.read(&HOSTED_KEYS)?,
store_virtual_keys: Some(snapshot.read(&STORE_VIRTUAL_KEYS)?),
prefix_for_stored_virtual_keys: snapshot.read(&PREFIX_FOR_STORED_VIRTUAL_KEYS)?,
access_mode,
primary_secret_name: snapshot.read(&PRIMARY_SECRET_NAME)?,
kms_key_id: snapshot.read(&KMS_KEY_ID)?,
custom_secret_manager: snapshot.read(&CUSTOM_SECRET_MANAGER)?,
aws_region_name: snapshot.read(&AWS_REGION_NAME)?,
aws_role_name: snapshot.read(&AWS_ROLE_NAME)?,
aws_session_name: snapshot.read(&AWS_SESSION_NAME)?,
aws_external_id: snapshot.read(&AWS_EXTERNAL_ID)?.map(SecretValue::new),
aws_profile_name: snapshot.read(&AWS_PROFILE_NAME)?,
aws_web_identity_token: snapshot
.read(&AWS_WEB_IDENTITY_TOKEN)?
.map(SecretValue::new),
aws_sts_endpoint: snapshot.read(&AWS_STS_ENDPOINT)?,
replica_regions: snapshot.read(&REPLICA_REGIONS)?,
..KeyManagementSettings::default()
};
let client = match snapshot.read(&CLIENT)? {
None => SecretManagerClient::Local,
Some(client) => SecretManagerClient::PythonCallback(client),
};
Ok(SecretManagerSnapshot {
client,
system,
settings,
settings_object: snapshot.read(&SETTINGS_OBJECT)?,
})
}
fn parse_optional_system(
field: &Field<'_>,
) -> Result<Option<KeyManagementSystem>, ProjectionError> {
let Some(value) = field.falsy_optional_string()? else {
return Ok(None);
};
serde_json::from_value(Value::String(value))
.map(Some)
.map_err(|error| {
ProjectionError::InvalidConfiguration(format!("secret manager system: {error}"))
})
}
fn parse_access_mode(field: &Field<'_>) -> Result<AccessMode, ProjectionError> {
let value = field.strict_string()?;
serde_json::from_value(Value::String(value)).map_err(|error| {
ProjectionError::InvalidConfiguration(format!("secret manager access mode: {error}"))
})
}
#[cfg(test)]
mod tests {
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
use super::{SecretManagerClient, project};
use crate::python_settings::PythonSettings;
fn snapshot<'py>(
py: Python<'py>,
system: &str,
access_mode: &str,
store_virtual_keys: Bound<'py, PyAny>,
hosted_keys: Bound<'py, PyAny>,
) -> crate::python_settings::Snapshot<'py> {
snapshot_with_client(
py,
system,
access_mode,
store_virtual_keys,
hosted_keys,
py.None().into_bound(py),
)
}
fn snapshot_with_client<'py>(
py: Python<'py>,
system: &str,
access_mode: &str,
store_virtual_keys: Bound<'py, PyAny>,
hosted_keys: Bound<'py, PyAny>,
client: Bound<'py, PyAny>,
) -> crate::python_settings::Snapshot<'py> {
let locals = PyDict::new(py);
locals.set_item("client", client).unwrap();
locals.set_item("system", system).unwrap();
locals.set_item("access_mode", access_mode).unwrap();
locals
.set_item("store_virtual_keys", store_virtual_keys)
.unwrap();
locals.set_item("hosted_keys", hosted_keys).unwrap();
py.run(
cr#"
from dataclasses import dataclass
from types import SimpleNamespace
@dataclass(frozen=True, slots=True)
class SecretManager:
system: object
access_mode: object
hosted_keys: object
primary_secret_name: object
store_virtual_keys: object
prefix_for_stored_virtual_keys: object
kms_key_id: object
custom_secret_manager: object
aws_region_name: object
aws_role_name: object
aws_session_name: object
aws_external_id: object
aws_profile_name: object
aws_web_identity_token: object
aws_sts_endpoint: object
replica_regions: object
client: object
settings_object: object
root = SimpleNamespace(secret_manager=SecretManager(
system=system,
access_mode=access_mode,
hosted_keys=hosted_keys,
primary_secret_name=None,
store_virtual_keys=store_virtual_keys,
prefix_for_stored_virtual_keys="litellm/",
kms_key_id=None,
custom_secret_manager=None,
aws_region_name=None,
aws_role_name=None,
aws_session_name=None,
aws_external_id=None,
aws_profile_name=None,
aws_web_identity_token=None,
aws_sts_endpoint=None,
replica_regions=None,
client=client,
settings_object=None,
))
"#,
Some(&locals),
Some(&locals),
)
.unwrap();
PythonSettings::SecretManagerBinding.snapshot(
locals
.get_item("root")
.unwrap()
.unwrap()
.getattr("secret_manager")
.unwrap(),
)
}
#[rstest::rstest]
#[case::string_true(Some("true"), false, true)]
#[case::string_one(Some("1"), false, true)]
#[case::true_value(None, true, true)]
#[case::false_value(None, false, false)]
#[case::string_false(Some("false"), false, true)]
fn python_compatible_boolean_coercion(
#[case] string_value: Option<&str>,
#[case] bool_value: bool,
#[case] expected: bool,
) {
Python::initialize();
Python::attach(|py| {
let store_virtual_keys = match string_value {
Some(value) => value.into_pyobject(py).unwrap().into_any(),
None => bool_value.into_pyobject(py).unwrap().to_owned().into_any(),
};
let hosted_keys = PyTuple::new(py, ["ONE"]).unwrap().into_any();
let projected = project(&snapshot(
py,
"local",
"read_only",
store_virtual_keys,
hosted_keys,
))
.unwrap();
assert_eq!(projected.settings.store_virtual_keys, Some(expected));
});
}
#[test]
fn unknown_system_is_rejected() {
Python::initialize();
Python::attach(|py| {
let error = project(&snapshot(
py,
"unknown",
"read_only",
false.into_pyobject(py).unwrap().to_owned().into_any(),
PyTuple::empty(py).into_any(),
))
.unwrap_err();
let error: PyErr = error.into();
assert!(error.is_instance_of::<pyo3::exceptions::PyValueError>(py));
});
}
#[test]
fn client_identity_selects_local_or_python_callback() {
Python::initialize();
Python::attach(|py| {
let falsy = false.into_pyobject(py).unwrap().to_owned().into_any();
let local = project(&snapshot(
py,
"local",
"read_only",
falsy.clone(),
PyTuple::empty(py).into_any(),
))
.unwrap();
assert!(matches!(local.client, SecretManagerClient::Local));
assert!(local.settings_object.is_none());
let manager = py.eval(c"object()", None, None).unwrap();
let custom = project(&snapshot_with_client(
py,
"custom",
"read_only",
falsy,
PyTuple::empty(py).into_any(),
manager.clone(),
))
.unwrap();
let SecretManagerClient::PythonCallback(client) = custom.client else {
panic!("a live client must stay a Python callback");
};
assert!(client.bind(py).is(&manager));
});
}
}

View file

@ -0,0 +1,3 @@
pub(crate) mod callback;
pub(crate) mod config;
pub(crate) mod resolved;

View file

@ -0,0 +1,252 @@
use std::{collections::HashMap, sync::Arc};
use futures_util::{future::BoxFuture, future::try_join_all};
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets};
use litellm_secrets::{
Error, FailurePolicy, OidcResolver, Secret, SecretManagerState, SecretResolver,
};
use super::config::SecretManagerSnapshot;
pub(crate) struct ResolvedSecrets {
resolver: SecretResolver,
}
impl ResolvedSecrets {
pub(crate) fn new(snapshot: SecretManagerSnapshot) -> Self {
Self::from_state(snapshot.into_state())
}
fn from_state(state: Arc<SecretManagerState>) -> Self {
Self {
resolver: SecretResolver::new(
state,
Arc::new(ProcessEnvironment),
OidcResolver::default(),
)
.with_failure_policy(FailurePolicy::EnvironmentFallback),
}
}
}
impl SecretSource for ResolvedSecrets {
fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result<Secrets, Error>> {
Box::pin(async move {
let values = try_join_all(names.iter().map(|name| async move {
self.resolver
.get_secret(name, None)
.await
.map(|secret| secret.map(|secret| ((*name).to_owned(), secret_value(secret))))
}))
.await?
.into_iter()
.flatten()
.collect::<HashMap<_, _>>();
Ok(Arc::new(ResolvedLookup { values }) as Secrets)
})
}
}
struct ResolvedLookup {
values: HashMap<String, String>,
}
impl Lookup for ResolvedLookup {
fn get(&self, name: &str) -> Option<String> {
self.values
.get(name)
.cloned()
.or_else(|| ProcessEnvironment.get(name))
}
}
fn secret_value(secret: Secret) -> String {
match secret {
Secret::String(value) => value.expose().to_owned(),
Secret::Bool(value) => if value { "True" } else { "False" }.to_owned(),
Secret::Json(value) => value.to_string(),
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use aws_sdk_secretsmanager::Client;
use aws_sdk_secretsmanager::config::{
BehaviorVersion, Credentials, Region, retry::RetryConfig,
};
use litellm_secrets::{AccessMode, KeyManagementSettings, SecretManager, SecretManagerState};
use litellm_secrets_aws::AwsSecretsManagerV2;
use serde_json::json;
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{body_partial_json, header},
};
use super::ResolvedSecrets;
use litellm_llms::base_llm::inference::secrets::SecretSource;
fn state(server: &MockServer, settings: KeyManagementSettings) -> Arc<SecretManagerState> {
let client = Client::from_conf(
aws_sdk_secretsmanager::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new("us-east-1"))
.credentials_provider(Credentials::new("test", "test", None, None, "test"))
.endpoint_url(server.uri())
.retry_config(RetryConfig::disabled())
.build(),
);
Arc::new(SecretManagerState::new(
SecretManager::AwsSecretsManagerV2(AwsSecretsManagerV2::new(
client,
(&settings).into(),
)),
settings,
))
}
async fn resolve(state: Arc<SecretManagerState>, name: &'static str) -> Option<String> {
ResolvedSecrets::from_state(state)
.resolve(&[name])
.await
.unwrap()
.get(name)
}
#[tokio::test]
async fn hosted_key_miss_falls_back_to_environment() {
let name = "LITELLM_RUST_BRIDGE_HOSTED_KEY_MISS";
unsafe { std::env::set_var(name, "env-key") };
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.and(body_partial_json(json!({"SecretId": name})))
.respond_with(
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})),
)
.expect(0)
.mount(&server)
.await;
let result = resolve(
state(
&server,
KeyManagementSettings {
hosted_keys: Some(vec!["OTHER".into()]),
..Default::default()
},
),
name,
)
.await;
unsafe { std::env::remove_var(name) };
assert_eq!(result.as_deref(), Some("env-key"));
assert_eq!(server.received_requests().await.unwrap().len(), 0);
}
#[tokio::test]
async fn manager_failure_falls_back_to_environment() {
let name = "LITELLM_RUST_BRIDGE_MANAGER_FAILURE";
unsafe { std::env::set_var(name, "env-key") };
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.respond_with(ResponseTemplate::new(500))
.expect(1)
.mount(&server)
.await;
let result = resolve(state(&server, KeyManagementSettings::default()), name).await;
unsafe { std::env::remove_var(name) };
assert_eq!(result.as_deref(), Some("env-key"));
assert_eq!(server.received_requests().await.unwrap().len(), 1);
let missing_server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.respond_with(ResponseTemplate::new(500))
.expect(1)
.mount(&missing_server)
.await;
let missing =
ResolvedSecrets::from_state(state(&missing_server, KeyManagementSettings::default()))
.resolve(&["LITELLM_RUST_BRIDGE_MANAGER_FAILURE_MISSING"])
.await;
assert!(matches!(missing, Err(litellm_secrets::Error::Aws(_))));
}
#[tokio::test]
async fn write_only_mode_never_consults_the_manager() {
let name = "LITELLM_RUST_BRIDGE_WRITE_ONLY";
unsafe { std::env::set_var(name, "env-key") };
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.respond_with(
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})),
)
.expect(0)
.mount(&server)
.await;
let result = resolve(
state(
&server,
KeyManagementSettings {
access_mode: AccessMode::WriteOnly,
..Default::default()
},
),
name,
)
.await;
unsafe { std::env::remove_var(name) };
assert_eq!(result.as_deref(), Some("env-key"));
assert_eq!(server.received_requests().await.unwrap().len(), 0);
}
#[tokio::test]
async fn read_only_mode_resolves_from_the_manager() {
let name = "LITELLM_RUST_BRIDGE_READ_ONLY";
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.and(body_partial_json(json!({"SecretId": name})))
.respond_with(
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})),
)
.expect(1)
.mount(&server)
.await;
assert_eq!(
resolve(state(&server, KeyManagementSettings::default()), name)
.await
.as_deref(),
Some("manager-key")
);
assert_eq!(server.received_requests().await.unwrap().len(), 1);
}
#[tokio::test]
async fn oidc_failures_are_not_converted_to_missing_secrets() {
let result = ResolvedSecrets::from_state(Arc::new(SecretManagerState::default()))
.resolve(&["oidc/"])
.await;
assert!(matches!(result, Err(litellm_secrets::Error::InvalidOidc)));
}
#[tokio::test]
async fn undeclared_names_still_read_the_process_environment() {
let name = "LITELLM_RUST_BRIDGE_UNDECLARED";
unsafe { std::env::set_var(name, "env-key") };
let server = MockServer::start().await;
let result = resolve(
state(
&server,
KeyManagementSettings {
hosted_keys: Some(vec!["OTHER".into()]),
..Default::default()
},
),
name,
)
.await;
unsafe { std::env::remove_var(name) };
assert_eq!(result.as_deref(), Some("env-key"));
assert_eq!(server.received_requests().await.unwrap().len(), 0);
}
}

View file

@ -1,158 +0,0 @@
use std::sync::Arc;
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use std::{num::NonZero, thread::available_parallelism};
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use litellm_host_python::release_gil;
use litellm_host_python::run_async;
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
types::PyAny,
};
use tokio::sync::Semaphore;
use crate::errors::RustBridgeDeclined;
/// Counts the input tokens of a raw request body off the Python event loop with
/// the GIL released. Python owns which requests get here and what to do with
/// the count. At most one encode per core runs at a time; the rest wait in the
/// async task, where a cancelled Python awaiter drops them before any blocking
/// work is scheduled.
#[pyclass(frozen)]
pub(crate) struct TokenCounter {
inner: Arc<CoreTokenCounter>,
encode_slots: Arc<Semaphore>,
}
#[pymethods]
impl TokenCounter {
#[new]
fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json))
}
#[cfg(all(not(feature = "fast"), feature = "huggingface"))]
{
Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
}
#[cfg(not(any(feature = "fast", feature = "huggingface")))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast or huggingface feature",
))
}
}
#[staticmethod]
fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
}
#[cfg(not(feature = "fast"))]
{
let _ = (py, rank_file);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast feature",
))
}
}
#[staticmethod]
fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
}
#[cfg(not(feature = "fast"))]
{
let _ = (py, rank_file);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast feature",
))
}
}
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
Self::load(py, || CoreTokenCounter::from_tiktoken(encoding))
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
let counter = Arc::clone(&self.inner);
let encode_slots = Arc::clone(&self.encode_slots);
let body = body.to_vec();
run_async(
py,
async move {
let _slot = encode_slots
.acquire_owned()
.await
.map_err(|error| Error::Task(error.to_string()))?;
tokio::task::spawn_blocking(move || count_body(&counter, &body))
.await
.map_err(|error| Error::Task(error.to_string()))?
},
token_count_error_to_pyerr,
)
}
}
impl TokenCounter {
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
fn load(
py: Python<'_>,
load: impl FnOnce() -> Result<CoreTokenCounter, Error> + Send,
) -> PyResult<Self> {
let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?;
Ok(Self {
inner: Arc::new(inner),
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
})
}
}
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
fn encode_parallelism() -> usize {
available_parallelism().map_or(1, NonZero::get)
}
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
let request = CountableRequest::parse(body)?;
counter.count_request(&request)
}
fn token_count_error_to_pyerr(error: Error) -> PyErr {
let message = error.to_string();
match error {
Error::Load(_)
| Error::Ranks(_)
| Error::UnicodeClasses
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
Error::RequestParse(_)
| Error::MissingInput
| Error::FloatText
| Error::ContentBlock
| Error::ArrayItems
| Error::JsonSerialization(_)
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
}
}

View file

@ -0,0 +1,713 @@
//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging
//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and
//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps.
use std::borrow::Cow;
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
use std::collections::HashMap;
use std::sync::Arc;
#[cfg(feature = "fast")]
use std::sync::OnceLock;
use litellm_host_python::{enter_native, release_gil};
#[cfg(feature = "fast")]
use litellm_token_counter::fast::{FastCounter, FastTokenizer};
use litellm_token_counter::{Error, TextCodec};
use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString};
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
use pyo3::exceptions::PyValueError;
#[cfg(feature = "huggingface")]
use pyo3::{exceptions::PyIOError, types::PyDict};
#[cfg(feature = "tiktoken")]
use pyo3::{
exceptions::{PyKeyError, PyRuntimeError},
types::PyBytes,
};
#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))]
use crate::errors::RustBridgeDeclined;
use crate::routes::token_counter::token_count_error_to_pyerr;
#[cfg(feature = "huggingface")]
use litellm_token_counter::huggingface::{
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy,
TruncationDirection, encoding_from_json, encoding_to_json,
};
#[cfg(feature = "tiktoken")]
use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary};
#[cfg(feature = "tiktoken")]
pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<TiktokenTokenizer> {
enter_native()?;
let resource: std::path::PathBuf =
PyModule::import(py, "litellm.litellm_core_utils.tokenizers")?
.getattr("__file__")?
.extract()?;
release_gil(py, || {
TiktokenTokenizer::from_cached_ranks(encoding, |file| {
std::fs::read_to_string(resource.with_file_name(file))
})
})
.map_err(|error| token_count_error_to_pyerr(error.into()))
}
pub(crate) enum Codec {
#[cfg(feature = "tiktoken")]
Tiktoken(TiktokenTokenizer),
#[cfg(feature = "huggingface")]
HuggingFace(HuggingFaceTokenizer),
}
impl Codec {
pub(crate) fn codec(&self) -> &dyn TextCodec {
match *self {
#[cfg(feature = "tiktoken")]
Self::Tiktoken(ref tokenizer) => tokenizer,
#[cfg(feature = "huggingface")]
Self::HuggingFace(ref tokenizer) => tokenizer,
}
}
#[cfg(feature = "fast")]
fn fast_counter(&self) -> Option<FastTokenizer> {
match *self {
#[cfg(feature = "tiktoken")]
Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(),
#[cfg(feature = "huggingface")]
Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(),
}
}
}
/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse,
/// and the opt-in count-only counter is derived from it once, on first use.
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Tokenizer {
inner: Arc<Codec>,
#[cfg(feature = "fast")]
fast: OnceLock<Option<Arc<FastTokenizer>>>,
}
#[pymethods]
impl Tokenizer {
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
let tokenizer = load_tiktoken(py, encoding)?;
Ok(Self::new(Codec::Tiktoken(tokenizer)))
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
#[staticmethod]
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
enter_native()?;
let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json))
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
Ok(Self::new(Codec::HuggingFace(tokenizer)))
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
#[staticmethod]
#[pyo3(signature = (identifier, revision = "main", token = None))]
fn from_pretrained(
py: Python<'_>,
identifier: &str,
revision: &str,
token: Option<&str>,
) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
enter_native()?;
let kwargs = PyDict::new(py);
kwargs.set_item("repo_id", identifier)?;
kwargs.set_item("filename", "tokenizer.json")?;
kwargs.set_item("revision", revision)?;
kwargs.set_item("token", token)?;
let path: String = PyModule::import(py, "huggingface_hub")?
.getattr("hf_hub_download")?
.call((), Some(&kwargs))?
.extract()?;
let json =
release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?;
Self::from_json(py, &json)
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, identifier, revision, token);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<Vec<u32>> {
enter_native()?;
let text = self.text(text)?;
release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (ids, skip_special_tokens = true))]
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
enter_native()?;
release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens))
.map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (text, fast = false))]
fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult<usize> {
enter_native()?;
let text = self.text(text)?;
let counter = self.counter(py, fast);
release_gil(py, || {
litellm_token_counter::Tokenizer::count_tokens(&counter, &text)
})
.map_err(token_count_error_to_pyerr)
}
#[getter]
fn name(&self) -> &str {
self.inner.codec().name()
}
// ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------
#[cfg(feature = "tiktoken")]
fn encode_special(
&self,
py: Python<'_>,
text: &Bound<'_, PyString>,
allowed: Vec<String>,
) -> PyResult<Vec<u32>> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let text = self.text(text)?;
release_gil(py, || tokenizer.encode_special(&text, &allowed))
.map_err(PyRuntimeError::new_err)
}
/// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`.
#[cfg(feature = "tiktoken")]
fn encode_with_unstable(
&self,
py: Python<'_>,
text: &Bound<'_, PyString>,
allowed: Vec<String>,
) -> PyResult<(Vec<u32>, Vec<Vec<u32>>)> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let text = self.text(text)?;
Ok(release_gil(py, || {
tokenizer.encode_with_unstable(&text, &allowed)
}))
}
/// The special tokens by text: tiktoken's `_special_tokens`.
#[cfg(feature = "tiktoken")]
fn special_tokens(&self) -> PyResult<HashMap<String, u32>> {
Ok(self
.vocabulary()?
.special_tokens()
.map(|(token, rank)| (token.to_owned(), rank))
.collect())
}
#[cfg(feature = "tiktoken")]
fn max_token_value(&self) -> PyResult<u32> {
Ok(self.vocabulary()?.max_token_value())
}
#[cfg(feature = "tiktoken")]
fn is_special_token(&self, token: u32) -> PyResult<bool> {
Ok(self.vocabulary()?.is_special_token(token))
}
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
#[cfg(feature = "tiktoken")]
fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyBytes>>> {
let vocabulary = self.vocabulary()?;
let values = release_gil(py, || vocabulary.token_byte_values());
Ok(values.iter().map(|value| PyBytes::new(py, value)).collect())
}
/// The token of one whole piece; `KeyError` when it is not in the vocabulary.
#[cfg(feature = "tiktoken")]
fn encode_single_token(&self, py: Python<'_>, piece: Vec<u8>) -> PyResult<u32> {
self.vocabulary()?
.encode_single_token(&piece)
.ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind()))
}
#[cfg(feature = "tiktoken")]
fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec<u32>) -> PyResult<Bound<'py, PyBytes>> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let bytes =
release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?;
Ok(PyBytes::new(py, &bytes))
}
// ---- Hugging Face: the `tokenizers.Tokenizer` surface -----------------------------------
#[cfg(feature = "huggingface")]
#[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_huggingface(
&self,
py: Python<'_>,
sequence: Sequence,
pair: Option<Sequence>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<HuggingFaceEncoding> {
enter_native()?;
let tokenizer = self.huggingface()?;
let sequence = sequence.input(is_pretokenized)?;
let input = match pair {
Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?),
None => EncodeInput::Single(sequence),
};
release_gil(py, || {
tokenizer.encode_result(input, add_special_tokens, fast)
})
.map(|inner| HuggingFaceEncoding { inner })
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_batch_huggingface(
&self,
py: Python<'_>,
inputs: Vec<(Sequence, Option<Sequence>)>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<Vec<HuggingFaceEncoding>> {
enter_native()?;
let tokenizer = self.huggingface()?;
let inputs = inputs
.into_iter()
.map(|(sequence, pair)| {
let sequence = sequence.input(is_pretokenized)?;
match pair {
Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)),
None => Ok(EncodeInput::Single(sequence)),
}
})
.collect::<PyResult<Vec<_>>>()?;
release_gil(py, || {
tokenizer.encode_batch_result(inputs, add_special_tokens, fast)
})
.map(|encodings| {
encodings
.into_iter()
.map(|inner| HuggingFaceEncoding { inner })
.collect()
})
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (pretty = false))]
fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult<String> {
enter_native()?;
let tokenizer = self.huggingface()?;
release_gil(py, || tokenizer.to_json(pretty))
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
fn token_to_id(&self, token: &str) -> PyResult<Option<u32>> {
Ok(self.huggingface()?.token_to_id(token))
}
#[cfg(feature = "huggingface")]
fn id_to_token(&self, id: u32) -> PyResult<Option<String>> {
Ok(self.huggingface()?.id_to_token(id))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (with_added_tokens = true))]
fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult<HashMap<String, u32>> {
let tokenizer = self.huggingface()?;
Ok(release_gil(py, || tokenizer.vocab(with_added_tokens)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (with_added_tokens = true))]
fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult<usize> {
Ok(self.huggingface()?.vocab_size(with_added_tokens))
}
/// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized,
/// special))`, for Python to rebuild as `tokenizers.AddedToken`.
#[cfg(feature = "huggingface")]
fn added_tokens_decoder(&self) -> PyResult<Vec<(u32, AddedTokenFields)>> {
Ok(self
.huggingface()?
.added_tokens_decoder()
.into_iter()
.map(|(id, token)| {
(
id,
(
token.content,
token.single_word,
token.lstrip,
token.rstrip,
token.normalized,
token.special,
),
)
})
.collect())
}
/// The padding parameters as `tokenizers.Tokenizer.padding` reports them.
#[cfg(feature = "huggingface")]
fn padding<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
let Some(params) = self.huggingface()?.padding() else {
return Ok(None);
};
let padding = PyDict::new(py);
padding.set_item(
"length",
match params.strategy {
PaddingStrategy::BatchLongest => None,
PaddingStrategy::Fixed(length) => Some(length),
},
)?;
padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?;
padding.set_item("pad_id", params.pad_id)?;
padding.set_item("pad_type_id", params.pad_type_id)?;
padding.set_item("pad_token", &params.pad_token)?;
padding.set_item("direction", params.direction.as_ref())?;
Ok(Some(padding))
}
/// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them.
#[cfg(feature = "huggingface")]
fn truncation<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
let Some(params) = self.huggingface()?.truncation() else {
return Ok(None);
};
let truncation = PyDict::new(py);
truncation.set_item("max_length", params.max_length)?;
truncation.set_item("stride", params.stride)?;
truncation.set_item("strategy", params.strategy.as_ref())?;
truncation.set_item("direction", params.direction.as_ref())?;
Ok(Some(truncation))
}
#[cfg(feature = "huggingface")]
fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult<usize> {
Ok(self.huggingface()?.num_special_tokens_to_add(is_pair))
}
#[cfg(feature = "huggingface")]
fn encode_special_tokens(&self) -> PyResult<bool> {
Ok(self.huggingface()?.encode_special_tokens())
}
}
#[cfg(feature = "huggingface")]
type AddedTokenFields = (String, bool, bool, bool, bool, bool);
impl Tokenizer {
fn new(inner: Codec) -> Self {
Self {
inner: Arc::new(inner),
#[cfg(feature = "fast")]
fast: OnceLock::new(),
}
}
pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter {
#[cfg(feature = "fast")]
if fast {
let counter = self.fast.get().unwrap_or_else(|| {
release_gil(py, || {
self.fast
.get_or_init(|| self.inner.fast_counter().map(Arc::new))
})
});
if let Some(counter) = counter {
return SharedCounter::Fast(Arc::clone(counter));
}
}
#[cfg(not(feature = "fast"))]
let _ = (py, fast);
SharedCounter::Codec(Arc::clone(&self.inner))
}
/// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode`
/// does; `tokenizers` rejects them, so that backend keeps the encode error.
fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult<Cow<'a, str>> {
match text.to_cow() {
Ok(text) => Ok(text),
Err(error) => match *self.inner {
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) if error.is_instance_of::<PyUnicodeEncodeError>(text.py()) => {
text.call_method1("encode", ("utf-16", "surrogatepass"))?
.call_method1("decode", ("utf-16", "replace"))?
.extract::<String>()
.map(Cow::Owned)
}
_ => Err(error),
},
}
}
#[cfg(feature = "tiktoken")]
fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> {
match *self.inner {
Codec::Tiktoken(ref tokenizer) => Ok(tokenizer),
#[cfg(feature = "huggingface")]
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
}
}
#[cfg(feature = "tiktoken")]
fn vocabulary(&self) -> PyResult<&Vocabulary> {
self.tiktoken()?.vocabulary().ok_or_else(|| {
PyRuntimeError::new_err("this encoding was built without its vocabulary")
})
}
#[cfg(feature = "huggingface")]
fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> {
match *self.inner {
Codec::HuggingFace(ref tokenizer) => Ok(tokenizer),
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
}
}
}
pub(crate) enum SharedCounter {
Codec(Arc<Codec>),
#[cfg(feature = "fast")]
Fast(Arc<FastTokenizer>),
}
impl litellm_token_counter::Tokenizer for SharedCounter {
fn count_tokens(&self, text: &str) -> Result<usize, Error> {
match self {
Self::Codec(codec) => codec.codec().count_tokens(text),
#[cfg(feature = "fast")]
Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from),
}
}
}
#[cfg(feature = "huggingface")]
#[derive(FromPyObject)]
pub(crate) enum Sequence {
Text(String),
Words(Vec<String>),
}
#[cfg(feature = "huggingface")]
impl Sequence {
fn input(self, is_pretokenized: bool) -> PyResult<InputSequence<'static>> {
match (self, is_pretokenized) {
(Self::Text(text), false) => Ok(text.into()),
(Self::Words(words), true) => Ok(words.into()),
_ => Err(pyo3::exceptions::PyTypeError::new_err(
"input must match is_pretokenized",
)),
}
}
}
#[cfg(feature = "huggingface")]
fn direction<T>(value: &str, left: T, right: T, what: &str) -> PyResult<T> {
match value {
"left" => Ok(left),
"right" => Ok(right),
other => Err(PyValueError::new_err(format!(
"invalid {what} direction {other:?}: expected 'left' or 'right'"
))),
}
}
/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id`
/// change it in place.
#[cfg(feature = "huggingface")]
#[pyclass(module = "litellm.rust_bridge._native")]
pub(crate) struct HuggingFaceEncoding {
inner: Encoding,
}
#[cfg(feature = "huggingface")]
#[pymethods]
impl HuggingFaceEncoding {
#[new]
#[pyo3(signature = (json = None))]
fn new(json: Option<&str>) -> PyResult<Self> {
let inner = match json {
Some(json) => encoding_from_json(json)
.map_err(|error| PyValueError::new_err(error.to_string()))?,
None => Encoding::default(),
};
Ok(Self { inner })
}
#[staticmethod]
#[pyo3(signature = (encodings, growing_offsets = true))]
fn merge(encodings: Vec<PyRef<'_, Self>>, growing_offsets: bool) -> Self {
Self {
inner: Encoding::merge(
encodings.iter().map(|encoding| encoding.inner.clone()),
growing_offsets,
),
}
}
fn __reduce__<'py>(
&self,
py: Python<'py>,
) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
let json = encoding_to_json(&self.inner)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok((py.get_type::<Self>(), (json,)))
}
fn __repr__(&self) -> String {
format!(
"Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \
attention_mask, special_tokens_mask, overflowing])",
self.inner.len()
)
}
fn __len__(&self) -> usize {
self.inner.len()
}
#[getter]
fn ids(&self) -> Vec<u32> {
self.inner.get_ids().to_vec()
}
#[getter]
fn tokens(&self) -> Vec<String> {
self.inner.get_tokens().to_vec()
}
#[getter]
fn offsets(&self) -> Vec<(usize, usize)> {
self.inner.get_offsets().to_vec()
}
#[getter]
fn type_ids(&self) -> Vec<u32> {
self.inner.get_type_ids().to_vec()
}
#[getter]
fn attention_mask(&self) -> Vec<u32> {
self.inner.get_attention_mask().to_vec()
}
#[getter]
fn special_tokens_mask(&self) -> Vec<u32> {
self.inner.get_special_tokens_mask().to_vec()
}
#[getter]
fn word_ids(&self) -> Vec<Option<u32>> {
self.inner.get_word_ids().to_vec()
}
#[getter]
fn sequence_ids(&self) -> Vec<Option<usize>> {
self.inner.get_sequence_ids()
}
#[getter]
fn overflowing(&self) -> Vec<Self> {
self.inner
.get_overflowing()
.iter()
.cloned()
.map(|inner| Self { inner })
.collect()
}
#[getter]
fn n_sequences(&self) -> usize {
self.inner.n_sequences()
}
#[pyo3(signature = (word_index, sequence_index = 0))]
fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
self.inner.word_to_tokens(word_index, sequence_index)
}
#[pyo3(signature = (word_index, sequence_index = 0))]
fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
self.inner.word_to_chars(word_index, sequence_index)
}
fn token_to_sequence(&self, token_index: usize) -> Option<usize> {
self.inner.token_to_sequence(token_index)
}
fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> {
self.inner
.token_to_chars(token_index)
.map(|(_, offsets)| offsets)
}
fn token_to_word(&self, token_index: usize) -> Option<u32> {
self.inner.token_to_word(token_index).map(|(_, word)| word)
}
#[pyo3(signature = (char_pos, sequence_index = 0))]
fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option<usize> {
self.inner.char_to_token(char_pos, sequence_index)
}
#[pyo3(signature = (char_pos, sequence_index = 0))]
fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option<u32> {
self.inner.char_to_word(char_pos, sequence_index)
}
fn set_sequence_id(&mut self, sequence_id: usize) {
self.inner.set_sequence_id(sequence_id);
}
#[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))]
fn pad(
&mut self,
length: usize,
direction: &str,
pad_id: u32,
pad_type_id: u32,
pad_token: &str,
) -> PyResult<()> {
let direction = self::direction(
direction,
PaddingDirection::Left,
PaddingDirection::Right,
"padding",
)?;
self.inner
.pad(length, pad_id, pad_type_id, pad_token, direction);
Ok(())
}
#[pyo3(signature = (max_length, stride = 0, direction = "right"))]
fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> {
let direction = self::direction(
direction,
TruncationDirection::Left,
TruncationDirection::Right,
"truncation",
)?;
self.inner.truncate(max_length, stride, direction);
Ok(())
}
}

View file

@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
use crate::SecretValue;
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum KeyManagementSystem {
GoogleKms,
@ -18,7 +18,7 @@ pub enum KeyManagementSystem {
Custom,
}
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum AccessMode {
#[default]
@ -33,7 +33,7 @@ impl AccessMode {
}
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)]
#[serde(default)]
pub struct KeyManagementSettings {
pub hosted_keys: Option<Vec<String>>,

View file

@ -24,6 +24,8 @@ pub enum Error {
OidcFile,
#[error("secret cannot be converted to {expected}")]
TypeMismatch { expected: &'static str },
#[error("external secret manager failed")]
ExternalManager(#[source] Box<dyn std::error::Error + Send + Sync>),
#[cfg(feature = "aws")]
#[error(transparent)]
Aws(#[from] litellm_secrets_aws::Error),

View file

@ -1,10 +1,24 @@
use std::{future::Future, pin::Pin, sync::Arc};
use litellm_core_utils::settings::Lookup;
use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue};
pub trait ExternalSecretManager: Send + Sync {
fn system(&self) -> KeyManagementSystem;
fn read_secret<'a>(
&'a self,
name: &'a str,
settings: &'a KeyManagementSettings,
environment: &'a (dyn Lookup + Send + Sync),
) -> Pin<Box<dyn Future<Output = Result<Option<Secret>, Error>> + Send + 'a>>;
}
#[derive(Clone)]
pub enum SecretManager {
Local,
External(Arc<dyn ExternalSecretManager>),
#[cfg(feature = "aws")]
AwsKms(crate::aws::AwsKms),
#[cfg(feature = "aws")]
@ -25,6 +39,7 @@ impl SecretManager {
pub fn system(&self) -> KeyManagementSystem {
match self {
Self::Local => KeyManagementSystem::Local,
Self::External(manager) => manager.system(),
#[cfg(feature = "aws")]
Self::AwsKms(_) => KeyManagementSystem::AwsKms,
#[cfg(feature = "aws")]
@ -54,6 +69,11 @@ pub async fn get_secret_from_manager(
.get(secret_name)
.map(SecretValue::new)
.map(Secret::String)),
SecretManager::External(manager) => {
manager
.read_secret(secret_name, _settings, environment)
.await
}
#[cfg(feature = "aws")]
SecretManager::AwsKms(client) => {
let ciphertext = environment

View file

@ -7,7 +7,7 @@ mod resolver;
mod state;
pub use error::Error;
pub use handler::{SecretManager, get_secret_from_manager};
pub use handler::{ExternalSecretManager, SecretManager, get_secret_from_manager};
pub use litellm_secrets_types::{
AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
};

View file

@ -72,6 +72,7 @@ impl SecretResolver {
Ok(value) => Ok(value
.or_else(|| self.environment_secret(name))
.or(default_value)),
Err(error @ Error::ExternalManager(_)) => Err(error),
Err(error) => match self.failure_policy {
FailurePolicy::Propagate => Err(error),
FailurePolicy::EnvironmentFallback => self

View file

@ -8,6 +8,8 @@ mod scanner;
mod tiktoken;
mod unicode_classes;
use std::sync::Arc;
use byte_level::ByteLevelCounter;
use scanner::{SplitPattern, TiktokenCounter};
@ -15,22 +17,29 @@ pub use error::Error;
enum Encoder {
HuggingFace {
tokenizer: Box<tokenizers::Tokenizer>,
tokenizer: Arc<tokenizers::Tokenizer>,
byte_level: Option<ByteLevelCounter>,
},
Tiktoken(TiktokenCounter),
}
/// A count-only tokenizer. Its model tables are immutable, so one built from an already
/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables.
pub struct FastTokenizer(Encoder);
impl FastTokenizer {
pub fn from_json(json: &str) -> Result<Self, Error> {
let tokenizer = json.parse::<tokenizers::Tokenizer>().map_err(Error::Load)?;
Ok(Self::from_shared(Arc::new(tokenizer)))
}
/// Counts with a Hugging Face model another codec already holds; nothing is re-parsed.
pub fn from_shared(tokenizer: Arc<tokenizers::Tokenizer>) -> Self {
let byte_level = ByteLevelCounter::detect(&tokenizer);
Ok(Self(Encoder::HuggingFace {
tokenizer: Box::new(tokenizer),
Self(Encoder::HuggingFace {
tokenizer,
byte_level,
}))
})
}
pub fn from_cl100k_ranks(ranks: &str) -> Result<Self, Error> {
@ -41,12 +50,36 @@ impl FastTokenizer {
Self::from_ranks(SplitPattern::O200k, ranks)
}
/// `cl100k_base` from ranks another loader already parsed.
pub fn from_cl100k_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::from_pairs(SplitPattern::Cl100k, pairs)
}
/// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks
/// another loader already parsed.
pub fn from_o200k_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::from_pairs(SplitPattern::O200k, pairs)
}
fn from_ranks(split: SplitPattern, ranks: &str) -> Result<Self, Error> {
TiktokenCounter::from_ranks(split, ranks)
.map(Encoder::Tiktoken)
.map(Self)
}
fn from_pairs<'a>(
split: SplitPattern,
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
TiktokenCounter::from_pairs(split, pairs)
.map(Encoder::Tiktoken)
.map(Self)
}
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
match &self.0 {
Encoder::Tiktoken(counter) => Ok(counter.count(text)),

View file

@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter {
impl TiktokenCounter {
pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result<Self, Error> {
Self::new(split, MergeRanks::parse(rank_file)?)
}
pub(super) fn from_pairs<'a>(
split: SplitPattern,
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::new(split, MergeRanks::from_pairs(pairs)?)
}
fn new(split: SplitPattern, ranks: MergeRanks) -> Result<Self, Error> {
Ok(Self {
ranks: MergeRanks::parse(rank_file)?,
ranks,
piece_len: split.piece_len(),
unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?,
})

View file

@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
impl MergeRanks {
pub(super) fn parse(text: &str) -> Result<Self, Error> {
let ranks = text
.lines()
.filter(|line| !line.is_empty())
.map(parse_line)
.collect::<Result<FxHashMap<_, _>, _>>()?;
Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line))
}
/// The same table from ranks another loader already parsed.
pub(super) fn from_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], Rank)>,
) -> Result<Self, Error> {
Self::from_entries(pairs.into_iter().map(|(bytes, rank)| {
if rank == NO_RANK {
return Err(Error::Ranks(format!("rank {rank} is reserved")));
}
Ok((Box::from(bytes), rank))
}))
}
fn from_entries(
entries: impl Iterator<Item = Result<(Box<[u8]>, Rank), Error>>,
) -> Result<Self, Error> {
let ranks = entries.collect::<Result<FxHashMap<_, _>, _>>()?;
if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
}

View file

@ -6,5 +6,6 @@ license.workspace = true
repository.workspace = true
[dependencies]
serde_json.workspace = true
thiserror.workspace = true
tokenizers.workspace = true

View file

@ -6,4 +6,6 @@ pub enum Error {
Load(#[source] tokenizers::Error),
#[error("tokenization failed: {0}")]
Encode(#[source] tokenizers::Error),
#[error("token decoding failed: {0}")]
Decode(#[source] tokenizers::Error),
}

View file

@ -2,22 +2,243 @@
mod error;
pub use error::Error;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
pub use error::Error;
use tokenizers::PostProcessor;
pub use tokenizers::{
AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams,
PaddingStrategy, TruncationDirection, TruncationParams,
};
pub fn encoding_from_json(json: &str) -> Result<Encoding, Error> {
serde_json::from_str(json).map_err(|error| Error::Load(error.into()))
}
pub fn encoding_to_json(encoding: &Encoding) -> Result<String, Error> {
serde_json::to_string(encoding).map_err(|error| Error::Load(error.into()))
}
pub struct HuggingFaceTokenizer {
tokenizer: Arc<tokenizers::Tokenizer>,
special_token_ids: HashSet<u32>,
}
impl HuggingFaceTokenizer {
pub fn from_json(json: &str) -> Result<Self, Error> {
json.parse::<tokenizers::Tokenizer>()
.map(Box::new)
.map(Self)
.map(Self::new)
.map_err(Error::Load)
}
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
let special_token_ids: HashSet<u32> = tokenizer
.get_added_tokens_decoder()
.into_iter()
.filter_map(|(id, token)| token.special.then_some(id))
.collect();
Self {
tokenizer: Arc::new(tokenizer),
special_token_ids,
}
}
/// The parsed model, for a count-only counter to share instead of parsing it again.
pub fn shared(&self) -> Arc<tokenizers::Tokenizer> {
Arc::clone(&self.tokenizer)
}
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
self.0
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.len())
.map_err(Error::Encode)
}
pub fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.get_ids().to_vec())
.map_err(Error::Encode)
}
pub fn encode_result<'a>(
&self,
input: EncodeInput<'a>,
add_special_tokens: bool,
fast: bool,
) -> Result<Encoding, Error> {
if fast {
return self
.tokenizer
.encode_fast(input, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_char_offsets(input, add_special_tokens)
.map_err(Error::Encode)
}
pub fn encode_batch_result<'a>(
&self,
inputs: Vec<EncodeInput<'a>>,
add_special_tokens: bool,
fast: bool,
) -> Result<Vec<Encoding>, Error> {
if fast {
return self
.tokenizer
.encode_batch_fast(inputs, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_batch_char_offsets(inputs, add_special_tokens)
.map_err(Error::Encode)
}
pub fn to_json(&self, pretty: bool) -> Result<String, Error> {
self.tokenizer.to_string(pretty).map_err(Error::Load)
}
pub fn token_to_id(&self, token: &str) -> Option<u32> {
self.tokenizer.token_to_id(token)
}
pub fn id_to_token(&self, id: u32) -> Option<String> {
self.tokenizer.id_to_token(id)
}
pub fn vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
self.tokenizer.get_vocab(with_added_tokens)
}
pub fn vocab_size(&self, with_added_tokens: bool) -> usize {
self.tokenizer.get_vocab_size(with_added_tokens)
}
/// The added tokens by id, in id order.
pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> {
let mut added: Vec<(u32, AddedToken)> = self
.tokenizer
.get_added_tokens_decoder()
.into_iter()
.collect();
added.sort_unstable_by_key(|(id, _)| *id);
added
}
pub fn padding(&self) -> Option<&PaddingParams> {
self.tokenizer.get_padding()
}
pub fn truncation(&self) -> Option<&TruncationParams> {
self.tokenizer.get_truncation()
}
/// How many special tokens the post-processor adds to a single sequence or a pair.
pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize {
self.tokenizer
.get_post_processor()
.map_or(0, |processor| processor.added_tokens(is_pair))
}
pub fn encode_special_tokens(&self) -> bool {
self.tokenizer.get_encode_special_tokens()
}
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
if !skip_special_tokens {
return self.tokenizer.decode(ids, false).map_err(Error::Decode);
}
let filtered_ids: Vec<u32> = ids
.iter()
.copied()
.filter(|id| !self.special_token_ids.contains(id))
.collect();
self.tokenizer
.decode(&filtered_ids, true)
.map_err(Error::Decode)
}
pub fn name(&self) -> &str {
"huggingface"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn codecs_round_trip_and_skip_special_tokens() {
let json = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
));
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
let ids = tokenizer.encode("<SOS>hello<EOT>").unwrap();
assert!(tokenizer.decode(&ids, false).unwrap().contains("<SOS>"));
assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello");
}
#[test]
fn decode_filters_special_added_tokens() {
let json = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 1,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {"type": "Whitespace"},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "<s>": 1, "hello": 2},
"unk_token": "<unk>"
}
}"#;
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("<s>"));
assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("<s>"));
}
#[test]
fn vocabulary_lookups_mirror_the_tokenizers_api() {
let json = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
));
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
let ids = tokenizer.encode("hello").unwrap();
let token = tokenizer.id_to_token(ids[0]).unwrap();
assert_eq!(tokenizer.token_to_id(&token), Some(ids[0]));
assert_eq!(tokenizer.id_to_token(u32::MAX), None);
assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true));
assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false));
let added = tokenizer.added_tokens_decoder();
assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0));
assert!(added.iter().any(|(_, token)| token.special));
assert!(tokenizer.padding().is_none());
assert!(tokenizer.truncation().is_none());
assert!(!tokenizer.encode_special_tokens());
assert_eq!(
tokenizer.num_special_tokens_to_add(false),
tokenizer.encode("").unwrap().len()
);
}
}

View file

@ -6,5 +6,8 @@ license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
once_cell = "1.21.3"
rustc-hash = "2.1.3"
thiserror.workspace = true
tiktoken-rs.workspace = true

View file

@ -1,27 +1,125 @@
#![forbid(unsafe_code)]
mod error;
mod ranks;
use std::collections::HashSet;
pub use error::UnsupportedTokenizer;
pub use ranks::{LoadError, Vocabulary};
pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE);
pub struct TiktokenTokenizer {
encoder: &'static tiktoken_rs::CoreBPE,
/// Present for encodings built from a rank file; the embedded tiktoken-rs singletons
/// behind [`from_name`](Self::from_name) keep their ranks private.
vocabulary: Option<&'static Vocabulary>,
name: &'static str,
}
impl TiktokenTokenizer {
/// Builds `name` from its packaged rank file (read through `load`), once per process.
/// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does.
pub fn from_cached_ranks(
name: &str,
load: impl FnOnce(&str) -> std::io::Result<String>,
) -> Result<Self, LoadError> {
let (loaded, name) = ranks::load(name, load)?;
Ok(Self {
encoder: &loaded.bpe,
vocabulary: Some(&loaded.vocabulary),
name,
})
}
/// The encodings tiktoken-rs embeds, for hosts without the packaged rank files.
pub fn from_name(name: &str) -> Result<Self, UnsupportedTokenizer> {
let tokenizer = match name {
"cl100k_base" => tiktoken_rs::cl100k_base_singleton(),
"o200k_base" => tiktoken_rs::o200k_base_singleton(),
"o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(),
"p50k_base" => tiktoken_rs::p50k_base_singleton(),
"p50k_edit" => tiktoken_rs::p50k_edit_singleton(),
"r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(),
let (encoder, name) = match name {
"cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"),
"o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"),
"o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"),
"p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"),
"p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"),
"r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"),
"gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"),
_ => return Err(UnsupportedTokenizer(name.to_owned())),
};
Ok(Self(tokenizer))
Ok(Self {
encoder,
vocabulary: None,
name,
})
}
pub fn vocabulary(&self) -> Option<&Vocabulary> {
self.vocabulary
}
pub fn count_tokens(&self, text: &str) -> usize {
self.0.count_ordinary(text)
self.encoder.count_ordinary(text)
}
pub fn encode(&self, text: &str) -> Vec<u32> {
self.encoder.encode_ordinary(text)
}
pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result<Vec<u32>, String> {
let allowed = allowed.iter().map(String::as_str).collect();
self.encoder
.encode(text, &allowed)
.map(|(ids, _)| ids)
.map_err(|error| error.to_string())
}
pub fn special_tokens(&self) -> HashSet<String> {
self.encoder
.special_tokens()
.into_iter()
.map(str::to_owned)
.collect()
}
/// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every
/// token sequence the unstable tail could still become, sorted for a stable order.
pub fn encode_with_unstable(
&self,
text: &str,
allowed: &[String],
) -> (Vec<u32>, Vec<Vec<u32>>) {
let allowed = allowed.iter().map(String::as_str).collect();
let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed);
let mut completions: Vec<Vec<u32>> = completions.into_iter().collect();
completions.sort_unstable();
(stable, completions)
}
pub fn decode_bytes(&self, ids: &[u32]) -> Result<Vec<u8>, String> {
self.encoder
.decode_bytes(ids)
.map_err(|error| error.to_string())
}
pub fn decode(&self, ids: &[u32]) -> Result<String, String> {
self.encoder
.decode_bytes(ids)
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
.map_err(|error| error.to_string())
}
pub fn name(&self) -> &str {
self.name
}
}
pub fn encoding_for_model(model: &str) -> Option<&'static str> {
match tiktoken_rs::tokenizer::get_tokenizer(model)? {
tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"),
tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"),
tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"),
tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"),
tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"),
tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => {
Some("r50k_base")
}
}
}
@ -66,5 +164,75 @@ mod tests {
panic!("unknown encoding must be rejected");
};
assert_eq!(name, "unknown-encoding");
assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2");
}
#[test]
fn codecs_round_trip_named_encodings() {
let encodings = [
"cl100k_base",
"o200k_base",
"o200k_harmony",
"p50k_base",
"p50k_edit",
"r50k_base",
"gpt2",
];
let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"];
for name in encodings {
let tokenizer = TiktokenTokenizer::from_name(name).unwrap();
for text in texts {
assert_eq!(
tokenizer.decode(&tokenizer.encode(text)).unwrap(),
text,
"{name}: {text:?}",
);
}
}
}
#[test]
fn decoding_token_prefixes_replaces_incomplete_utf8() {
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
let reference = tiktoken_rs::cl100k_base_singleton();
let ids = tokenizer.encode("🙂漢字");
for end in 1..=ids.len() {
let bytes = reference.decode_bytes(&ids[..end]).unwrap();
assert_eq!(
tokenizer.decode(&ids[..end]).unwrap(),
String::from_utf8_lossy(&bytes),
);
}
assert!(tokenizer.decode(&[u32::MAX]).is_err());
}
#[test]
fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() {
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
let text = "hello fanta";
let (stable, completions) = tokenizer.encode_with_unstable(text, &[]);
assert!(
text.as_bytes()
.starts_with(&tokenizer.decode_bytes(&stable).unwrap())
);
assert!(!completions.is_empty());
for completion in &completions {
let mut ids = stable.clone();
ids.extend(completion);
assert!(
tokenizer
.decode_bytes(&ids)
.unwrap()
.starts_with(text.as_bytes())
);
}
assert!(completions.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn encoding_for_model_maps_known_models() {
assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base"));
assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base"));
assert_eq!(encoding_for_model("unknown-model"), None);
}
}

View file

@ -0,0 +1,340 @@
use base64::{Engine, engine::general_purpose::STANDARD};
use once_cell::sync::OnceCell;
use rustc_hash::FxHashMap;
use thiserror::Error;
use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank};
use crate::UnsupportedTokenizer;
const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4";
const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790";
const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15";
const LEGACY_PATTERN: &str =
r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s";
const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s";
static CL100K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static O200K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static HARMONY_ENCODER: OnceCell<Loaded> = OnceCell::new();
static P50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static EDIT_ENCODER: OnceCell<Loaded> = OnceCell::new();
static R50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built
/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does
/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`).
pub(super) struct Loaded {
pub(super) bpe: CoreBPE,
pub(super) vocabulary: Vocabulary,
}
/// The byte-level vocabulary of a tiktoken encoding.
pub struct Vocabulary {
ranks: FxHashMap<Vec<u8>, Rank>,
special_tokens: FxHashMap<String, Rank>,
max_token_value: Rank,
}
impl Vocabulary {
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
pub fn token_byte_values(&self) -> Vec<Vec<u8>> {
let mut values: Vec<Vec<u8>> = self.ranks.keys().cloned().collect();
values.sort_unstable();
values
}
/// The rank of one whole token: a mergeable piece first, then a special token's text.
pub fn encode_single_token(&self, piece: &[u8]) -> Option<Rank> {
if let Some(rank) = self.ranks.get(piece) {
return Some(*rank);
}
std::str::from_utf8(piece)
.ok()
.and_then(|text| self.special_tokens.get(text).copied())
}
pub fn max_token_value(&self) -> Rank {
self.max_token_value
}
/// Every mergeable token with its rank, for building other tables from one parse.
pub fn ranks(&self) -> impl Iterator<Item = (&[u8], Rank)> + '_ {
self.ranks
.iter()
.map(|(bytes, rank)| (bytes.as_slice(), *rank))
}
/// The special tokens with their ranks, tiktoken's `_special_tokens`.
pub fn special_tokens(&self) -> impl Iterator<Item = (&str, Rank)> + '_ {
self.special_tokens
.iter()
.map(|(token, rank)| (token.as_str(), *rank))
}
pub fn is_special_token(&self, rank: Rank) -> bool {
self.special_tokens.values().any(|special| *special == rank)
}
}
#[derive(Debug, Error)]
pub enum LoadError {
#[error(transparent)]
Unsupported(#[from] UnsupportedTokenizer),
#[error("failed to load tiktoken ranks: {0}")]
Ranks(String),
}
/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays
/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share
/// one cached encoder.
pub(super) fn load(
name: &str,
load_file: impl FnOnce(&str) -> std::io::Result<String>,
) -> Result<(&'static Loaded, &'static str), LoadError> {
let (requested, canonical, file, cache) = match name {
"cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER),
"o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER),
"o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER),
"p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER),
"p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER),
"r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER),
"gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER),
_ => return Err(UnsupportedTokenizer(name.to_owned()).into()),
};
let loaded = cache.get_or_try_init(|| {
let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?;
build(canonical, &ranks)
})?;
Ok((loaded, requested))
}
fn build(name: &str, ranks: &str) -> Result<Loaded, LoadError> {
let parsed = ranks
.lines()
.map(parse_rank)
.collect::<Result<Vec<_>, _>>()?;
let encoder: FxHashMap<_, _> = parsed
.into_iter()
.filter(|(_, rank)| name != "r50k_base" || *rank < 50256)
.collect();
if encoder
.values()
.collect::<std::collections::HashSet<_>>()
.len()
!= encoder.len()
|| (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..]))
{
return Err(LoadError::Ranks("invalid vocabulary ranks".into()));
}
let (pattern, specials): (&str, &[(&str, Rank)]) = match name {
"cl100k_base" => (
CL100K_PATTERN,
&[
("<|endoftext|>", 100257),
("<|fim_prefix|>", 100258),
("<|fim_middle|>", 100259),
("<|fim_suffix|>", 100260),
("<|endofprompt|>", 100276),
],
),
"o200k_base" => (
O200K_BASE_PAT_STR,
&[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)],
),
"o200k_harmony" => (
O200K_BASE_PAT_STR,
&[
("<|startoftext|>", 199998),
("<|endoftext|>", 199999),
("<|reserved_200000|>", 200000),
("<|reserved_200001|>", 200001),
("<|return|>", 200002),
("<|constrain|>", 200003),
("<|reserved_200004|>", 200004),
("<|channel|>", 200005),
("<|start|>", 200006),
("<|end|>", 200007),
("<|message|>", 200008),
("<|reserved_200009|>", 200009),
("<|reserved_200010|>", 200010),
("<|reserved_200011|>", 200011),
("<|call|>", 200012),
],
),
"p50k_edit" => (
LEGACY_PATTERN,
&[
("<|endoftext|>", 50256),
("<|fim_prefix|>", 50281),
("<|fim_middle|>", 50282),
("<|fim_suffix|>", 50283),
],
),
_ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]),
};
let reserved = (200013..=201087)
.filter(|_| name == "o200k_harmony")
.map(|rank| (format!("<|reserved_{rank}|>"), rank));
let special_tokens: FxHashMap<String, Rank> = specials
.iter()
.map(|(token, rank)| ((*token).to_owned(), *rank))
.chain(reserved)
.collect();
let max_token_value = encoder
.values()
.chain(special_tokens.values())
.copied()
.max()
.ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?;
let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern)
.map_err(|error| LoadError::Ranks(error.to_string()))?;
Ok(Loaded {
bpe,
vocabulary: Vocabulary {
ranks: encoder,
special_tokens,
max_token_value,
},
})
}
fn parse_rank(line: &str) -> Result<(Vec<u8>, Rank), LoadError> {
let (token, rank) = line
.split_once(' ')
.ok_or_else(|| LoadError::Ranks("missing rank".into()))?;
let bytes = STANDARD
.decode(token)
.map_err(|error| LoadError::Ranks(error.to_string()))?;
let rank = rank
.parse()
.map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?;
Ok((bytes, rank))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::TiktokenTokenizer;
fn read_packaged_ranks(file: &str) -> std::io::Result<String> {
std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../litellm/litellm_core_utils/tokenizers")
.join(file),
)
}
#[test]
fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() {
for name in [
"cl100k_base",
"o200k_base",
"o200k_harmony",
"p50k_base",
"p50k_edit",
"r50k_base",
"gpt2",
] {
if name != "gpt2" {
assert!(
TiktokenTokenizer::from_cached_ranks(name, |_| {
Err(std::io::Error::other("unreadable vocabulary"))
})
.is_err()
);
}
let loads = std::sync::atomic::AtomicUsize::new(0);
let barrier = std::sync::Barrier::new(4);
let encoders = std::thread::scope(|scope| {
let tasks: Vec<_> = (0..4)
.map(|_| {
scope.spawn(|| {
barrier.wait();
TiktokenTokenizer::from_cached_ranks(name, |file| {
loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
read_packaged_ranks(file)
})
.unwrap()
})
})
.collect();
tasks
.into_iter()
.map(|task| task.join().unwrap())
.collect::<Vec<_>>()
});
assert_eq!(loads.into_inner(), usize::from(name != "gpt2"));
let actual = &encoders[0];
let expected = TiktokenTokenizer::from_name(name).unwrap();
assert_eq!(actual.special_tokens(), expected.special_tokens());
let specials: Vec<_> = expected.special_tokens().into_iter().collect();
let special_text = specials.join(" ");
assert_eq!(
actual.encode_special(&special_text, &specials).unwrap(),
expected.encode_special(&special_text, &specials).unwrap()
);
for text in [
"",
"café 漢字 ع 🙂",
"a\r\nb\t ",
" hello 123456789",
&special_text,
] {
let ids = expected.encode(text);
assert_eq!(actual.encode(text), ids, "{name}: {text:?}");
assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}");
assert_eq!(
actual.decode_bytes(&ids).unwrap(),
expected.decode_bytes(&ids).unwrap()
);
}
let cached = TiktokenTokenizer::from_cached_ranks(name, |_| {
panic!("reloaded cached vocabulary")
})
.unwrap();
assert_eq!(cached.encode("cached"), expected.encode("cached"));
assert_eq!(cached.name(), name);
assert!(expected.vocabulary().is_none());
assert_vocabulary_lookups(name, actual);
}
}
/// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the
/// encoder itself and against the known vocabulary sizes.
fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) {
let max_token_value = match name {
"cl100k_base" => 100_276,
"o200k_base" => 200_018,
"o200k_harmony" => 201_087,
"p50k_base" => 50_280,
"p50k_edit" => 50_283,
"r50k_base" | "gpt2" => 50_256,
_ => unreachable!("{name}"),
};
let vocabulary = tokenizer.vocabulary().unwrap();
assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}");
let values = vocabulary.token_byte_values();
assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}");
for piece in values.iter().step_by(997) {
let rank = vocabulary.encode_single_token(piece).unwrap();
assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}");
assert!(!vocabulary.is_special_token(rank), "{name}");
}
for (token, rank) in vocabulary.special_tokens() {
assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank));
assert!(vocabulary.is_special_token(rank), "{name}: {token}");
}
assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None);
}
#[test]
fn malformed_ranks_return_errors_instead_of_panicking() {
for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] {
assert!(build("cl100k_base", ranks).is_err());
}
let repeated_rank = (0..=u8::MAX)
.map(|byte| format!("{} 0\n", STANDARD.encode([byte])))
.collect::<String>();
assert!(build("cl100k_base", &repeated_rank).is_err());
}
}

View file

@ -1,6 +1,10 @@
# Token counting
`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface
`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer`
Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing
`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it
The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation
@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t
The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2`
All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend
Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits

View file

@ -32,6 +32,8 @@ pub enum Error {
JsonUtf8(#[source] FromUtf8Error),
#[error("tokenization failed: {0}")]
Encode(#[source] Box<dyn std::error::Error + Send + Sync>),
#[error("token decoding failed: {0}")]
Decode(String),
#[error("token counting task failed: {0}")]
Task(String),
}

View file

@ -1,7 +1,31 @@
use litellm_token_counter_fast::Error as BackendError;
pub use litellm_token_counter_fast::FastTokenizer;
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
pub trait FastCounter: TextCodec {
fn fast_counter(&self) -> Option<FastTokenizer>;
}
#[cfg(feature = "huggingface")]
impl FastCounter for crate::huggingface::HuggingFaceTokenizer {
fn fast_counter(&self) -> Option<FastTokenizer> {
Some(FastTokenizer::from_shared(self.shared()))
}
}
#[cfg(feature = "tiktoken")]
impl FastCounter for crate::tiktoken::TiktokenTokenizer {
fn fast_counter(&self) -> Option<FastTokenizer> {
let vocabulary = self.vocabulary()?;
match self.name() {
"cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()),
"o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()),
_ => return None,
}
.ok()
}
}
impl TokenCounter {
pub fn from_json_fast(tokenizer_json: &str) -> Result<Self, Error> {
@ -39,3 +63,65 @@ impl From<BackendError> for Error {
}
}
}
#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))]
mod tests {
use super::*;
use crate::huggingface::HuggingFaceTokenizer;
use crate::tiktoken::TiktokenTokenizer;
const TEXTS: [&str; 4] = [
"",
"hello world <|endoftext|>",
"café 漢字 ع 🙂 line\r\n indented 123456789",
"<SOS>system<EOT> a\u{301} fi",
];
fn packaged(file: &str) -> String {
std::fs::read_to_string(format!(
"{}/../../../litellm/litellm_core_utils/tokenizers/{file}",
env!("CARGO_MANIFEST_DIR")
))
.unwrap()
}
#[test]
fn fast_counters_derived_from_codecs_count_like_the_codecs() {
let huggingface =
HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap();
let fast = huggingface.fast_counter().unwrap();
for text in TEXTS {
assert_eq!(
fast.count_tokens(text).unwrap(),
Tokenizer::count_tokens(&huggingface, text).unwrap(),
"{text:?}"
);
}
for name in ["cl100k_base", "o200k_base", "o200k_harmony"] {
let tiktoken =
TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap();
let fast = tiktoken.fast_counter().unwrap();
for text in TEXTS {
assert_eq!(
fast.count_tokens(text).unwrap(),
tiktoken.count_tokens(text),
"{name}: {text:?}"
);
}
}
}
#[test]
fn encodings_without_a_fast_scanner_keep_the_codec() {
let tiktoken =
TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap();
assert!(tiktoken.fast_counter().is_none());
assert!(
TiktokenTokenizer::from_name("cl100k_base")
.unwrap()
.fast_counter()
.is_none()
);
}
}

View file

@ -1,7 +1,11 @@
use litellm_token_counter_huggingface::Error as BackendError;
pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
pub use litellm_token_counter_huggingface::{
AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection,
PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json,
encoding_to_json,
};
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
impl TokenCounter {
pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer {
}
}
impl TextCodec for HuggingFaceTokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
HuggingFaceTokenizer::encode(self, text).map_err(Error::from)
}
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from)
}
fn name(&self) -> &str {
HuggingFaceTokenizer::name(self)
}
}
impl From<BackendError> for Error {
fn from(error: BackendError) -> Self {
match error {
BackendError::Load(source) => Self::Load(source),
BackendError::Encode(source) => Self::Encode(source),
BackendError::Decode(source) => Self::Decode(source.to_string()),
}
}
}

View file

@ -20,5 +20,5 @@ pub mod tiktoken;
pub use counter::{InputTokenCount, TokenCounter};
pub use error::Error;
pub use tokenizer::Tokenizer;
pub use tokenizer::{TextCodec, Tokenizer};
pub use types::CountableRequest;

View file

@ -1,7 +1,7 @@
pub use litellm_token_counter_tiktoken::TiktokenTokenizer;
use litellm_token_counter_tiktoken::UnsupportedTokenizer;
use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer};
pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model};
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
impl TokenCounter {
pub fn from_tiktoken(encoding: &str) -> Result<Self, Error> {
@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer {
}
}
impl TextCodec for TiktokenTokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
Ok(TiktokenTokenizer::encode(self, text))
}
fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result<String, Error> {
TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string()))
}
fn name(&self) -> &str {
TiktokenTokenizer::name(self)
}
}
impl From<UnsupportedTokenizer> for Error {
fn from(error: UnsupportedTokenizer) -> Self {
Self::UnsupportedTokenizer(error.0)
}
}
impl From<LoadError> for Error {
fn from(error: LoadError) -> Self {
match error {
LoadError::Unsupported(error) => error.into(),
LoadError::Ranks(message) => Self::Ranks(message),
}
}
}

View file

@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync {
fn count_tokens(&self, text: &str) -> Result<usize, Error>;
}
pub trait TextCodec: Tokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error>;
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error>;
fn name(&self) -> &str;
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -58,7 +58,8 @@ from ._lazy_imports_registry import (
if TYPE_CHECKING:
import httpx
from tiktoken import Encoding
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
def get_litellm_globals() -> dict[str, object]:
@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
# These are special lazy loaders for things that are used internally
# They're separate from the main lazy import system because they have specific use cases
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
_default_encoding: "Encoding | None" = None
def _get_default_encoding() -> "Tokenizer":
from litellm.rust_bridge.tokenizer import get_encoding
def _get_default_encoding() -> "Encoding":
"""
Lazily load and cache the default OpenAI encoding.
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
at `litellm` import time. The encoding is cached after the first import.
This is used internally by utils.py functions that need the encoding but shouldn't
trigger its import during module load.
"""
global _default_encoding
if _default_encoding is None:
from litellm.litellm_core_utils.default_encoding import encoding
_default_encoding = encoding
return _default_encoding
return get_encoding("cl100k_base")
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time

View file

@ -4,7 +4,7 @@ from types import MappingProxyType
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
from litellm import main
from litellm.rust_bridge.catalog import Context, Delivery, Route
from litellm.rust_bridge.catalog import Delivery, Route, RouteContext
from litellm.rust_bridge.chat_completions.entrypoints import (
NATIVE_ACOMPLETION,
NATIVE_COMPLETION,
@ -72,8 +72,8 @@ def _public_request(
)
def _context(request: LiteLLMChatCompletionsRequest) -> Context:
return Context(
def _context(request: LiteLLMChatCompletionsRequest) -> RouteContext:
return RouteContext(
Route.CHAT_COMPLETIONS,
provider=request.custom_llm_provider,
model=request.model,

View file

@ -323,6 +323,9 @@ class OCRPricing(TypedDict, total=False):
annotation_cost_per_page: ReadOnly[float | None]
_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"})
def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool:
return (
(model_info.get("input_cost_per_token") or 0.0) > 0
@ -331,6 +334,11 @@ def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool:
)
def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool:
mode: Final = model_info.get("mode")
return mode is None or mode in _WALL_CLOCK_PRICED_MODES
def _per_second_pricing_cost(
model: str,
custom_llm_provider: str | None,
@ -340,7 +348,7 @@ def _per_second_pricing_cost(
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
if _has_token_or_tiered_pricing(model_info):
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
return None
input_cost_per_second: Final = model_info.get("input_cost_per_second")
output_cost_per_second: Final = model_info.get("output_cost_per_second")

View file

@ -427,7 +427,7 @@ class LLMCallSpanData:
# plain ``.get`` — no repeated ``isinstance`` guards.
raw_response: Final = payload.get("response")
response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {})
choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response)
choices_out: Final = _output_choices(response)
# ``finish_reasons`` is metadata, not content, so derive it from
# ``choices_out`` before gating. The raw message/choice bodies are only
# retained when content capture is enabled (see ``capture_span_content``);
@ -752,20 +752,99 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return (choice,)
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
markdowns: Final = tuple(
text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None
def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
"""The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes."""
return (
_completion_choices(response)
or _responses_choices(response)
or _ocr_choices(response)
or _transcription_choices(response)
or _moderation_choices(response)
or _image_choices(response)
or _binary_choices(response)
)
if not markdowns:
def _text_choice(content: str, finish_reason: str | None = None) -> _Choice:
message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}
return {"message": message, "finish_reason": finish_reason}
def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]:
return (_text_choice("\n\n".join(parts)),) if parts else ()
def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
return tuple(
_text_choice(text, as_str(choice.get("finish_reason")))
if "message" not in choice and isinstance(text := choice.get("text"), str)
else choice
for choice in _dicts(response.get("choices"))
)
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None)
)
def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
text: Final = response.get("text")
return (_text_choice(text),) if isinstance(text, str) and text else ()
def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(
_moderation_verdict(flagged, result.get("categories"))
for result in _dicts(response.get("results"))
if isinstance(flagged := result.get("flagged"), bool)
)
)
def _moderation_verdict(flagged: bool, categories: object) -> str:
if not flagged:
return "not flagged"
hits: Final = (
tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True)
if isinstance(categories, dict)
else ()
)
return f"flagged: {', '.join(hits)}" if hits else "flagged"
def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None)
)
def _image_summary(item: Mapping[str, object]) -> str | None:
location: Final = _image_location(item)
if location is None:
return None
revised: Final = as_str(item.get("revised_prompt"))
return f"{revised}\n{location}" if revised else location
def _image_location(item: Mapping[str, object]) -> str | None:
url: Final = as_str(item.get("url"))
if url is not None:
return url
encoded: Final = item.get("b64_json")
if not isinstance(encoded, str):
return None
return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)"
def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
size: Final = as_int(response.get("num_bytes"))
if size is None:
return ()
message: Final[_AssistantMessage] = {
"role": "assistant",
"content": "\n\n".join(markdowns),
"refusal": None,
"tool_calls": None,
}
choice: Final[_Choice] = {"message": message, "finish_reason": None}
return (choice,)
content_type: Final = as_str(response.get("content_type"))
return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),)
def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None:

View file

@ -6,8 +6,9 @@ Core files:
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
- `default_encoding.py`: code for loading the default encoding (tiktoken)
- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.

Some files were not shown because too many files have changed in this diff Show more