mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'origin/main' into litellm_per_second_pricing_responses_messages
# Conflicts: # litellm/cost_calculator.py # tests/test_litellm/test_cost_calculator.py
This commit is contained in:
commit
e411bc4444
306 changed files with 13362 additions and 3671 deletions
10
.github/actions/cache-cargo-build/action.yml
vendored
10
.github/actions/cache-cargo-build/action.yml
vendored
|
|
@ -15,6 +15,12 @@ description: >-
|
|||
cache the same directory for different workloads, and a shared key would let
|
||||
whichever ran first deny the others a save.
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: "Cargo profile the build uses (dev or release)"
|
||||
required: false
|
||||
default: "dev"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
|
|
@ -25,6 +31,6 @@ runs:
|
|||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
litellm-rust/target
|
||||
key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
|
||||
key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maturin-dev-
|
||||
${{ runner.os }}-maturin-${{ inputs.profile }}-
|
||||
|
|
|
|||
1
.github/e2e-stack/select_tests.py
vendored
1
.github/e2e-stack/select_tests.py
vendored
|
|
@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile(
|
|||
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
|
||||
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
|
||||
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
|
||||
)
|
||||
HARNESS: Final = re.compile(
|
||||
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
|
||||
|
|
|
|||
14
.github/scripts/verify_linux_native_wheel.py
vendored
14
.github/scripts/verify_linux_native_wheel.py
vendored
|
|
@ -134,7 +134,16 @@ def main(
|
|||
uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
|
||||
native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
|
||||
native_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
native_path.write_bytes(archive.read(native_member))
|
||||
native_bytes: Final = archive.read(native_member)
|
||||
native_path.write_bytes(native_bytes)
|
||||
duplicated_vocabularies: Final = tuple(
|
||||
member.filename
|
||||
for member in wheel_members
|
||||
if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
|
||||
and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
|
||||
and member.file_size > 0
|
||||
and archive.read(member) in native_bytes
|
||||
)
|
||||
|
||||
wheel_metadata_tags_match: Final = (
|
||||
len(wheel_metadata_tags) == len(expanded_filename_tags)
|
||||
|
|
@ -205,7 +214,7 @@ def main(
|
|||
native_module: Final = load_native_module(native_path)
|
||||
native_module_loads: Final = native_module is not None
|
||||
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
|
||||
native_size_limit: Final = 40_000_000
|
||||
native_size_limit: Final = 35_000_000
|
||||
native_size_within_limit: Final = native_member.file_size <= native_size_limit
|
||||
validations: Final = (
|
||||
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
|
||||
|
|
@ -223,6 +232,7 @@ def main(
|
|||
("Native module loads", native_module_loads),
|
||||
("Production module omits the panic test hook", panic_test_hook_absent),
|
||||
(f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
|
||||
("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
|
||||
("Wheel contents are valid", not unexpected_members),
|
||||
)
|
||||
|
||||
|
|
|
|||
36
.github/workflows/codspeed.yml
vendored
36
.github/workflows/codspeed.yml
vendored
|
|
@ -13,6 +13,7 @@ on:
|
|||
- ".github/workflows/codspeed.yml"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/actions/cache-cargo-build/**"
|
||||
- ".github/scripts/uv_sync_with_retries.sh"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
|
@ -25,6 +26,7 @@ on:
|
|||
- ".github/workflows/codspeed.yml"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/actions/cache-cargo-build/**"
|
||||
- ".github/scripts/uv_sync_with_retries.sh"
|
||||
# Allow CodSpeed to trigger backtest performance analysis
|
||||
# in order to generate initial data
|
||||
workflow_dispatch:
|
||||
|
|
@ -59,19 +61,27 @@ jobs:
|
|||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
with:
|
||||
profile: release
|
||||
|
||||
# Build the wheel and resolve every dependency outside the CodSpeed
|
||||
# runner: the same maturin build took 42 minutes inside `codspeed run`
|
||||
# versus under 3 minutes as a plain step (LIT-6183)
|
||||
- name: Build environment
|
||||
- name: Build the release wheel
|
||||
run: uv build --wheel --out-dir dist
|
||||
|
||||
- name: Install the wheel into the benchmark environment
|
||||
run: |
|
||||
UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
|
||||
uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
|
||||
|
||||
- name: Collect benchmarks
|
||||
env:
|
||||
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
|
||||
LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
|
||||
run: >
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
||||
uv run --frozen --no-default-groups
|
||||
--with pytest==8.3.5
|
||||
--with pytest-codspeed==4.3.0
|
||||
--with "mcp>=2.2.0,<3.0"
|
||||
--with "a2a-sdk>=1.1.0,<2.0"
|
||||
pytest
|
||||
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
||||
--import-mode=importlib
|
||||
-p pytest_codspeed.plugin
|
||||
tests/benchmarks/
|
||||
--codspeed
|
||||
|
|
@ -82,13 +92,9 @@ jobs:
|
|||
with:
|
||||
mode: simulation
|
||||
run: >
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
||||
uv run --frozen --no-default-groups
|
||||
--with pytest==8.3.5
|
||||
--with pytest-codspeed==4.3.0
|
||||
--with "mcp>=2.2.0,<3.0"
|
||||
--with "a2a-sdk>=1.1.0,<2.0"
|
||||
pytest
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
|
||||
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
||||
--import-mode=importlib
|
||||
-p pytest_codspeed.plugin
|
||||
tests/benchmarks/
|
||||
--codspeed
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--extra saml \
|
||||
--python python3.13
|
||||
|
||||
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
"""Guard the cost map on pull requests.
|
||||
|
||||
Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
|
||||
and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
|
||||
litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
|
||||
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
|
||||
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
|
||||
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
|
||||
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
|
||||
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
|
||||
files and may only add or update models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
)
|
||||
|
||||
|
||||
def touches_cost_map(changed_files: Sequence[str]) -> bool:
|
||||
return any(path in GUARDED_PATHS for path in changed_files)
|
||||
|
||||
|
||||
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
|
||||
if bot:
|
||||
return "bot contract enforced"
|
||||
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
|
||||
|
||||
|
||||
def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
|
||||
if not bot and not touches_cost_map(changed_files):
|
||||
return ()
|
||||
head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
|
||||
if isinstance(head_map, str):
|
||||
return (head_map,)
|
||||
return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
|
||||
|
||||
|
||||
def _git(*args: str) -> str:
|
||||
def _git(*args: str) -> str | None:
|
||||
result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
|
||||
return result.stdout if result.returncode == 0 else ""
|
||||
return result.stdout if result.returncode == 0 else None
|
||||
|
||||
|
||||
def snapshot(revision: str) -> Snapshot:
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
|
||||
|
||||
|
||||
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
|
||||
diff: Final = _git("diff", "--name-only", "--no-renames", base, head)
|
||||
return None if diff is None else tuple(diff.splitlines())
|
||||
|
||||
|
||||
def main(argv: Sequence[str]) -> int:
|
||||
|
|
@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int:
|
|||
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
|
||||
args: Final = parser.parse_args(argv)
|
||||
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
|
||||
changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
|
||||
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
|
||||
contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
|
||||
changed: Final = changed_files(args.base, args.head)
|
||||
if changed is None:
|
||||
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
|
||||
return 1
|
||||
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
|
||||
contract: Final = contract_for(bot, changed)
|
||||
if failures:
|
||||
print(f"cost map guard failed ({contract}):")
|
||||
print("\n".join(f"- {failure}" for failure in failures))
|
||||
|
|
|
|||
|
|
@ -108,6 +108,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
"/cursor/",
|
||||
"/milvus/",
|
||||
"/openai_passthrough/",
|
||||
"/tinyfish/",
|
||||
# Dynamic provider / toolset passthrough (path templates)
|
||||
"/{provider}/",
|
||||
"/toolset/",
|
||||
|
|
|
|||
11
litellm-rust/Cargo.lock
generated
11
litellm-rust/Cargo.lock
generated
|
|
@ -2894,6 +2894,7 @@ dependencies = [
|
|||
"litellm-host",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
"litellm-secrets",
|
||||
"litellm-types",
|
||||
"mime_guess",
|
||||
"moka",
|
||||
|
|
@ -3006,6 +3007,7 @@ dependencies = [
|
|||
"litellm-framing",
|
||||
"litellm-host",
|
||||
"litellm-http",
|
||||
"litellm-secrets",
|
||||
"litellm-types",
|
||||
"reqwest 0.12.28",
|
||||
"rstest",
|
||||
|
|
@ -3024,6 +3026,7 @@ dependencies = [
|
|||
name = "litellm-python-bridge"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"aws-sdk-secretsmanager",
|
||||
"bytes",
|
||||
"criterion",
|
||||
"futures-util",
|
||||
|
|
@ -3047,6 +3050,9 @@ dependencies = [
|
|||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
"litellm-secrets",
|
||||
"litellm-secrets-aws",
|
||||
"litellm-secrets-types",
|
||||
"litellm-token-counter",
|
||||
"litellm-types",
|
||||
"pyo3",
|
||||
|
|
@ -3062,6 +3068,7 @@ dependencies = [
|
|||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"url",
|
||||
"wiremock",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3246,6 +3253,7 @@ dependencies = [
|
|||
name = "litellm-token-counter-huggingface"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokenizers",
|
||||
]
|
||||
|
|
@ -3254,6 +3262,9 @@ dependencies = [
|
|||
name = "litellm-token-counter-tiktoken"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"once_cell",
|
||||
"rustc-hash",
|
||||
"thiserror 2.0.19",
|
||||
"tiktoken-rs",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ veil = "0.3.0"
|
|||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = "thin"
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
panic = "unwind"
|
||||
debug = false
|
||||
|
|
|
|||
|
|
@ -621,6 +621,26 @@ mod tests {
|
|||
None
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_names_cover_environment_reads() {
|
||||
let seen = std::sync::Arc::new(std::sync::Mutex::new(
|
||||
std::collections::BTreeSet::<String>::new(),
|
||||
));
|
||||
let recorded = seen.clone();
|
||||
let env = |name: &str| {
|
||||
recorded.lock().unwrap().insert(name.to_string());
|
||||
None
|
||||
};
|
||||
resolve_aws_region(None, &Map::new(), &env);
|
||||
aws_auth_config(&Map::new(), &env);
|
||||
assert!(
|
||||
seen.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|name| crate::constants::SECRET_NAMES.contains(&name.as_str()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_region_comes_from_the_call_then_the_model_then_the_environment() {
|
||||
let params = Map::from_iter([("aws_region_name".to_string(), Value::from("eu-west-1"))]);
|
||||
|
|
|
|||
|
|
@ -14,6 +14,19 @@ pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
|
|||
pub const AWS_STS_ENDPOINT: &str = "AWS_STS_ENDPOINT";
|
||||
pub const AWS_EXTERNAL_ID: &str = "AWS_EXTERNAL_ID";
|
||||
pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";
|
||||
pub const SECRET_NAMES: &[&str] = &[
|
||||
AWS_ACCESS_KEY_ID,
|
||||
AWS_SECRET_ACCESS_KEY,
|
||||
AWS_SESSION_TOKEN,
|
||||
AWS_REGION_NAME,
|
||||
AWS_REGION,
|
||||
AWS_SESSION_NAME,
|
||||
AWS_PROFILE_NAME,
|
||||
AWS_ROLE_NAME,
|
||||
AWS_WEB_IDENTITY_TOKEN,
|
||||
AWS_STS_ENDPOINT,
|
||||
AWS_EXTERNAL_ID,
|
||||
];
|
||||
|
||||
/// Headers SigV4 covers, beyond the `x-amz-` / `x-amzn-` prefixes. Mirrors
|
||||
/// Python's `_filter_headers_for_aws_signature` allowlist.
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@ mod native;
|
|||
mod resolve;
|
||||
mod types;
|
||||
|
||||
pub use resolve::AzureAuthService;
|
||||
pub use resolve::{AzureAuthService, SECRET_NAMES};
|
||||
pub use types::{AzureAuthInputs, ConfigValue};
|
||||
|
|
|
|||
|
|
@ -19,6 +19,17 @@ const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST";
|
|||
const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL";
|
||||
const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE";
|
||||
|
||||
pub const SECRET_NAMES: &[&str] = &[
|
||||
AZURE_AD_TOKEN_ENV,
|
||||
AZURE_TENANT_ID_ENV,
|
||||
AZURE_CLIENT_ID_ENV,
|
||||
AZURE_CLIENT_SECRET_ENV,
|
||||
AZURE_SCOPE_ENV,
|
||||
AZURE_AUTHORITY_HOST_ENV,
|
||||
AZURE_CREDENTIAL_ENV,
|
||||
AZURE_FEDERATED_TOKEN_FILE_ENV,
|
||||
];
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) enum AzureCredentialPlan {
|
||||
Supplied(Sourced<ResolvedCredential>),
|
||||
|
|
@ -440,13 +451,14 @@ fn non_empty_reference(value: &str, kind: &str) -> Result<String, Error> {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
use std::future::Future;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use super::{
|
||||
AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, oidc_reference,
|
||||
AzureAuthService, AzureCredentialPlan, AzureTokenAcquirer, SECRET_NAMES, oidc_reference,
|
||||
resolve_reference, select_auth_plan,
|
||||
};
|
||||
use crate::native::ValidatedAzureRequest;
|
||||
|
|
@ -517,6 +529,24 @@ mod tests {
|
|||
assert!(matches!(plan, AzureCredentialPlan::Native(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_names_cover_environment_reads() {
|
||||
let seen = std::sync::Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
|
||||
let recorded = seen.clone();
|
||||
let inputs = AzureAuthInputs::default();
|
||||
select_auth_plan(&inputs, &|name| {
|
||||
recorded.lock().unwrap().insert(name.to_string());
|
||||
None
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
seen.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|name| SECRET_NAMES.contains(&name.as_str()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn supplied_token_does_not_require_refresh() {
|
||||
let params = json!({"azure_ad_token": "token"});
|
||||
|
|
|
|||
|
|
@ -23,6 +23,16 @@ const VERTEXAI_PROJECT_ENV: &str = "VERTEXAI_PROJECT";
|
|||
const VERTEXAI_LOCATION_ENV: &str = "VERTEXAI_LOCATION";
|
||||
const VERTEX_LOCATION_ENV: &str = "VERTEX_LOCATION";
|
||||
|
||||
pub const SECRET_NAMES: &[&str] = &[
|
||||
VERTEX_AI_API_KEY_ENV,
|
||||
VERTEXAI_API_KEY_ENV,
|
||||
VERTEXAI_CREDENTIALS_ENV,
|
||||
GOOGLE_APPLICATION_CREDENTIALS_ENV,
|
||||
VERTEXAI_PROJECT_ENV,
|
||||
VERTEXAI_LOCATION_ENV,
|
||||
VERTEX_LOCATION_ENV,
|
||||
];
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct VertexConfig {
|
||||
credentials: Option<Sourced<SecretValue>>,
|
||||
|
|
@ -406,6 +416,7 @@ fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use serde_json::json;
|
||||
|
|
@ -476,6 +487,27 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn secret_names_cover_environment_reads() {
|
||||
let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
|
||||
let recorded = seen.clone();
|
||||
let env = |name: &str| {
|
||||
recorded.lock().unwrap().insert(name.to_string());
|
||||
None
|
||||
};
|
||||
let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
|
||||
auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
|
||||
.await
|
||||
.unwrap();
|
||||
get_vertex_ai_location(&VertexConfig::default(), &env);
|
||||
assert!(
|
||||
seen.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|name| SECRET_NAMES.contains(&name.as_str()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_primary_values_fall_back_to_python_aliases() {
|
||||
let config = config(json!({
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use serde::Deserialize;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use veil::Redact;
|
||||
|
||||
#[derive(Redact, Clone, Deserialize)]
|
||||
|
|
@ -23,6 +24,12 @@ impl PartialEq for SecretValue {
|
|||
|
||||
impl Eq for SecretValue {}
|
||||
|
||||
impl Hash for SecretValue {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
self.0.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::SecretValue;
|
||||
|
|
|
|||
|
|
@ -36,6 +36,8 @@ Callers supply Unix time for response freshness. Backend TTL uses its own clock.
|
|||
|
||||
The extension keeps a private test harness for memory and Redis single and batch response lookup and storage. Batch lookup returns ordered values plus missing indices for embedding partial-hit wiring. No bridge-only cache type is part of the public API
|
||||
|
||||
The bridge also exposes a production-shaped response cache runtime selected through the Rust catalog. Its shipped rule set is empty, so current SDK, Router, and proxy calls stay on Python and do not construct native cache resources. Tests can inject a rule and build the native memory runtime from an ordinary Python `Cache` configuration without changing the legacy cache classes
|
||||
|
||||
Object responses are written as they are, and every other response shape is written as a serialized string, which is the pair of shapes Python reads. A string on the wire is therefore always a serialized response, so string-valued responses round trip. Typed backends such as memory never pass through the codec
|
||||
|
||||
The resolver reads the namespace's `cache` attribute each time it resolves. A captured binding retains the selected service for its operation, including background writes. `None` disables caching. Custom Python cache objects keep their original methods, arguments, returned awaitables, exceptions, and caller-task execution
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use std::{sync::Mutex, time::Duration};
|
||||
|
||||
use litellm_cache::{BaseCache, Error, ExactCacheContext};
|
||||
use litellm_cache::Error;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{CacheEntry, ResponseCache, ResponseCacheRequest};
|
||||
use crate::{ExactResponseCache, ResponseCacheRequest};
|
||||
|
||||
pub struct WriteBuffer {
|
||||
flush_size: usize,
|
||||
|
|
@ -18,9 +18,9 @@ impl WriteBuffer {
|
|||
}
|
||||
}
|
||||
|
||||
pub async fn async_store<B: BaseCache<Value = CacheEntry, Context = ExactCacheContext>>(
|
||||
pub async fn async_store(
|
||||
&self,
|
||||
cache: &ResponseCache<B>,
|
||||
cache: &dyn ExactResponseCache,
|
||||
request: &ResponseCacheRequest,
|
||||
response: Value,
|
||||
now: Duration,
|
||||
|
|
|
|||
148
litellm-rust/crates/cache-response/src/exact.rs
Normal file
148
litellm-rust/crates/cache-response/src/exact.rs
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
use std::{future::Future, pin::Pin, time::Duration};
|
||||
|
||||
use litellm_cache::{
|
||||
BaseCache, BatchCache, CacheConnectionResult, Error, ExactCacheContext, FlushCache,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{CacheEntry, PartialHits, ResponseCache, ResponseCacheRequest};
|
||||
|
||||
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
/// Object-safe view of a `ResponseCache` over an exact-match backend, so hosts can hold every
|
||||
/// exact backend behind one pointer without erasing which backend it is elsewhere.
|
||||
pub trait ExactResponseCache: Send + Sync {
|
||||
fn default_ttl(&self) -> Option<Duration>;
|
||||
|
||||
fn lookup(&self, request: &ResponseCacheRequest, now: Duration)
|
||||
-> Result<Option<Value>, Error>;
|
||||
|
||||
fn store(
|
||||
&self,
|
||||
request: &ResponseCacheRequest,
|
||||
response: Value,
|
||||
now: Duration,
|
||||
) -> Result<(), Error>;
|
||||
|
||||
fn lookup_batch(
|
||||
&self,
|
||||
requests: &[ResponseCacheRequest],
|
||||
now: Duration,
|
||||
) -> Result<PartialHits, Error>;
|
||||
|
||||
fn async_lookup<'a>(
|
||||
&'a self,
|
||||
request: &'a ResponseCacheRequest,
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<Option<Value>, Error>>;
|
||||
|
||||
fn async_store<'a>(
|
||||
&'a self,
|
||||
request: &'a ResponseCacheRequest,
|
||||
response: Value,
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<(), Error>>;
|
||||
|
||||
fn async_lookup_batch<'a>(
|
||||
&'a self,
|
||||
requests: &'a [ResponseCacheRequest],
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<PartialHits, Error>>;
|
||||
|
||||
fn async_store_batch<'a>(
|
||||
&'a self,
|
||||
entries: Vec<(ResponseCacheRequest, Value)>,
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<(), Error>>;
|
||||
|
||||
fn async_store_entries<'a>(
|
||||
&'a self,
|
||||
entries: Vec<(ResponseCacheRequest, Value, Duration)>,
|
||||
) -> BoxFuture<'a, Result<(), Error>>;
|
||||
|
||||
fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>>;
|
||||
|
||||
fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result<CacheConnectionResult, Error>>;
|
||||
}
|
||||
|
||||
impl<B> ExactResponseCache for ResponseCache<B>
|
||||
where
|
||||
B: BaseCache<Value = CacheEntry, Context = ExactCacheContext> + BatchCache + FlushCache,
|
||||
{
|
||||
fn default_ttl(&self) -> Option<Duration> {
|
||||
ResponseCache::default_ttl(self)
|
||||
}
|
||||
|
||||
fn lookup(
|
||||
&self,
|
||||
request: &ResponseCacheRequest,
|
||||
now: Duration,
|
||||
) -> Result<Option<Value>, Error> {
|
||||
ResponseCache::lookup(self, request, now)
|
||||
}
|
||||
|
||||
fn store(
|
||||
&self,
|
||||
request: &ResponseCacheRequest,
|
||||
response: Value,
|
||||
now: Duration,
|
||||
) -> Result<(), Error> {
|
||||
ResponseCache::store(self, request, response, now)
|
||||
}
|
||||
|
||||
fn lookup_batch(
|
||||
&self,
|
||||
requests: &[ResponseCacheRequest],
|
||||
now: Duration,
|
||||
) -> Result<PartialHits, Error> {
|
||||
ResponseCache::lookup_batch(self, requests, now)
|
||||
}
|
||||
|
||||
fn async_lookup<'a>(
|
||||
&'a self,
|
||||
request: &'a ResponseCacheRequest,
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<Option<Value>, Error>> {
|
||||
Box::pin(ResponseCache::async_lookup(self, request, now))
|
||||
}
|
||||
|
||||
fn async_store<'a>(
|
||||
&'a self,
|
||||
request: &'a ResponseCacheRequest,
|
||||
response: Value,
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<(), Error>> {
|
||||
Box::pin(ResponseCache::async_store(self, request, response, now))
|
||||
}
|
||||
|
||||
fn async_lookup_batch<'a>(
|
||||
&'a self,
|
||||
requests: &'a [ResponseCacheRequest],
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<PartialHits, Error>> {
|
||||
Box::pin(ResponseCache::async_lookup_batch(self, requests, now))
|
||||
}
|
||||
|
||||
fn async_store_batch<'a>(
|
||||
&'a self,
|
||||
entries: Vec<(ResponseCacheRequest, Value)>,
|
||||
now: Duration,
|
||||
) -> BoxFuture<'a, Result<(), Error>> {
|
||||
Box::pin(ResponseCache::async_store_batch(self, entries, now))
|
||||
}
|
||||
|
||||
fn async_store_entries<'a>(
|
||||
&'a self,
|
||||
entries: Vec<(ResponseCacheRequest, Value, Duration)>,
|
||||
) -> BoxFuture<'a, Result<(), Error>> {
|
||||
Box::pin(ResponseCache::async_store_entries(self, entries))
|
||||
}
|
||||
|
||||
fn async_flush<'a>(&'a self) -> BoxFuture<'a, Result<(), Error>> {
|
||||
Box::pin(ResponseCache::async_flush(self))
|
||||
}
|
||||
|
||||
fn test_connection<'a>(&'a self) -> BoxFuture<'a, Result<CacheConnectionResult, Error>> {
|
||||
Box::pin(ResponseCache::test_connection(self))
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ mod buffer;
|
|||
mod caching;
|
||||
mod codec;
|
||||
mod embedding;
|
||||
mod exact;
|
||||
mod response;
|
||||
|
||||
pub use buffer::WriteBuffer;
|
||||
|
|
@ -11,4 +12,5 @@ pub use caching::{
|
|||
};
|
||||
pub use codec::ResponseCacheCodec;
|
||||
pub use embedding::PartialHits;
|
||||
pub use exact::ExactResponseCache;
|
||||
pub use response::{ResponseCache, ResponseCacheRequest};
|
||||
|
|
|
|||
|
|
@ -17,6 +17,11 @@ pub fn parse_str_bool(value: &str) -> Option<bool> {
|
|||
token.eq_ignore_ascii_case("false").then_some(false)
|
||||
}
|
||||
|
||||
/// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true.
|
||||
pub fn parse_redis_bool(value: &str) -> bool {
|
||||
value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes")
|
||||
}
|
||||
|
||||
impl<'de> DeserializeAs<'de, i64> for LaxI64 {
|
||||
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
|
||||
deserializer.deserialize_any(Self)
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ url.workspace = true
|
|||
veil.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-secrets.workspace = true
|
||||
litellm-auth-gcp.workspace = true
|
||||
litellm-llms = { workspace = true, features = ["test-support"] }
|
||||
rstest.workspace = true
|
||||
|
|
|
|||
|
|
@ -22,7 +22,12 @@ pub(crate) async fn perform_ocr_request(
|
|||
) -> Result<LiteLLMOcrResponse, Error> {
|
||||
request.response_format()?;
|
||||
let config = request.config;
|
||||
let request = prepare_request(request, caller_document, client);
|
||||
let secrets = client
|
||||
.secret_source()
|
||||
.resolve(&config.secret_names())
|
||||
.await
|
||||
.map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
|
||||
let request = prepare_request(request, caller_document, client, secrets);
|
||||
let hooks = OcrCallHooks::new(host.clone(), &request, config);
|
||||
config.ocr(client, &request, &hooks).await
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
use litellm_auth::{InputSource, SecretValue, Sourced};
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
handler::OcrClient,
|
||||
transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
|
||||
use litellm_llms::base_llm::{
|
||||
inference::secrets::Secrets,
|
||||
ocr::{
|
||||
handler::OcrClient,
|
||||
transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
|
||||
},
|
||||
};
|
||||
|
||||
use super::provider_config::OcrProvider;
|
||||
|
|
@ -11,6 +14,7 @@ pub(crate) fn prepare_request(
|
|||
request: ResolvedOcrRequest,
|
||||
caller_document: bool,
|
||||
client: &OcrClient,
|
||||
secrets: Secrets,
|
||||
) -> PreparedOcrRequest {
|
||||
let credentials = request.credentials.clone();
|
||||
let (preferred_api_key_env, api_base_env) = match request.config.provider() {
|
||||
|
|
@ -24,7 +28,7 @@ pub(crate) fn prepare_request(
|
|||
| OcrProvider::Reducto
|
||||
| OcrProvider::VertexAi => (None, None),
|
||||
};
|
||||
let secret = |name: &str| client.secrets().truthy(name);
|
||||
let secret = |name: &str| secrets.truthy(name);
|
||||
let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
|
||||
credentials.api_key.clone().or_else(|| {
|
||||
preferred_api_key_env
|
||||
|
|
@ -60,12 +64,7 @@ pub(crate) fn prepare_request(
|
|||
PreparedOcrRequest {
|
||||
model,
|
||||
document,
|
||||
connection: OcrConnection::new(
|
||||
resolved,
|
||||
transport,
|
||||
client.settings().clone(),
|
||||
client.secrets().clone(),
|
||||
),
|
||||
connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
|
||||
caller_document,
|
||||
optional_params,
|
||||
input_sources,
|
||||
|
|
@ -79,6 +78,7 @@ pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedO
|
|||
request,
|
||||
true,
|
||||
&OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()),
|
||||
std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -114,6 +114,10 @@ impl OcrConfigKind {
|
|||
with_config!(self, config => config.get_api_key_env_var())
|
||||
}
|
||||
|
||||
pub(crate) fn secret_names(self) -> Vec<&'static str> {
|
||||
with_config!(self, config => config.secret_names())
|
||||
}
|
||||
|
||||
pub(crate) fn get_health_check_document(self) -> OcrDocument {
|
||||
with_config!(self, config => config.get_health_check_document())
|
||||
}
|
||||
|
|
@ -213,6 +217,8 @@ fn is_document_intelligence_model(model: &str) -> bool {
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashSet;
|
||||
|
||||
use litellm_auth::{InputSource, Sourced};
|
||||
use litellm_llms::{
|
||||
base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
|
||||
|
|
@ -221,6 +227,27 @@ mod tests {
|
|||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case(OcrConfigKind::AwsTextract)]
|
||||
#[case(OcrConfigKind::AwsTextractAnalyze)]
|
||||
#[case(OcrConfigKind::Cohere)]
|
||||
#[case(OcrConfigKind::Mistral)]
|
||||
#[case(OcrConfigKind::AzureAi)]
|
||||
#[case(OcrConfigKind::AzureCohere)]
|
||||
#[case(OcrConfigKind::AzureDocumentIntelligence)]
|
||||
#[case(OcrConfigKind::ReductoLegacy)]
|
||||
#[case(OcrConfigKind::ReductoV3)]
|
||||
#[case(OcrConfigKind::VertexAi)]
|
||||
#[case(OcrConfigKind::VertexDeepSeek)]
|
||||
fn secret_names_include_api_keys_without_duplicates(#[case] config: OcrConfigKind) {
|
||||
let names = config.secret_names();
|
||||
let unique = names.iter().collect::<HashSet<_>>();
|
||||
assert_eq!(names.len(), unique.len());
|
||||
if let Some(api_key) = config.get_api_key_env_var() {
|
||||
assert!(names.contains(&api_key));
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("cohere")]
|
||||
#[case("mistral")]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_auth_gcp::VertexAuth;
|
||||
use litellm_host::{
|
||||
event::{CallEvent, MachineEvent, WireRequest},
|
||||
|
|
@ -10,11 +11,14 @@ use litellm_http::{
|
|||
HttpClientPool, HttpSettings, Resolution,
|
||||
media::{PublicDnsResolver, UrlPolicy},
|
||||
};
|
||||
use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets};
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
error::Error as OcrError,
|
||||
handler::OcrClient,
|
||||
settings::OcrSettings,
|
||||
transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig},
|
||||
transformation::{
|
||||
BaseOcrConfig, LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig,
|
||||
},
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::{Value, json};
|
||||
|
|
@ -27,6 +31,32 @@ use super::{
|
|||
};
|
||||
use crate::ocr::route::{LocalOcrHost, OcrOp, OcrOpResult, ocr_machine};
|
||||
|
||||
struct RecordingSecretSource {
|
||||
names: Arc<Mutex<Vec<&'static str>>>,
|
||||
values: &'static [(&'static str, &'static str)],
|
||||
api_base: String,
|
||||
}
|
||||
|
||||
impl SecretSource for RecordingSecretSource {
|
||||
fn resolve<'a>(
|
||||
&'a self,
|
||||
names: &'a [&'static str],
|
||||
) -> BoxFuture<'a, Result<Secrets, litellm_secrets::Error>> {
|
||||
*self.names.lock().unwrap() = names.to_vec();
|
||||
let values = self.values;
|
||||
let api_base = self.api_base.clone();
|
||||
Box::pin(async move {
|
||||
Ok(Arc::new(move |name: &str| match name {
|
||||
"MISTRAL_AZURE_API_BASE" => Some(api_base.clone()),
|
||||
_ => values
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string()),
|
||||
}) as Secrets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::mistral("mistral/model", json!({}))]
|
||||
#[case::vertex("vertex_ai/mistral-ocr-latest", json!({"vertex_project":"test-project", "vertex_location":"us-central1"}))]
|
||||
|
|
@ -184,14 +214,11 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source(
|
|||
#[case] expected_key: &str,
|
||||
) {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
let secret_base = base.clone();
|
||||
let client = ocr_client().with_secrets(Arc::new(move |name: &str| match name {
|
||||
"MISTRAL_AZURE_API_BASE" => Some(secret_base.clone()),
|
||||
"MISTRAL_API_BASE" => Some("http://127.0.0.1:9/never-read".into()),
|
||||
_ => secrets
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| value.to_string()),
|
||||
let names = Arc::new(Mutex::new(Vec::new()));
|
||||
let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource {
|
||||
names: names.clone(),
|
||||
values: secrets,
|
||||
api_base: base.clone(),
|
||||
}));
|
||||
let request = decode_request(OcrWireRequest {
|
||||
model: "mistral/model".into(),
|
||||
|
|
@ -208,9 +235,47 @@ async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source(
|
|||
|
||||
crate::ocr::client::perform(&client, request).await.unwrap();
|
||||
server.await.unwrap();
|
||||
assert_eq!(
|
||||
*names.lock().unwrap(),
|
||||
litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names()
|
||||
);
|
||||
assert!(seen.lock().unwrap()[0].contains(&format!("authorization: Bearer {expected_key}")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mistral_ocr_resolves_provider_secrets_before_transformation() {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
let names = Arc::new(Mutex::new(Vec::new()));
|
||||
let client = ocr_client().with_secrets(Arc::new(RecordingSecretSource {
|
||||
names: names.clone(),
|
||||
values: &[("MISTRAL_API_KEY", "source-key")],
|
||||
api_base: base.clone(),
|
||||
}));
|
||||
let request = decode_request(OcrWireRequest {
|
||||
model: "mistral/mistral-ocr-latest".into(),
|
||||
document: json!({
|
||||
"type":"document_url",
|
||||
"document_url":"data:application/pdf;base64,YWJj"
|
||||
}),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
extra_headers: None,
|
||||
optional_params: Default::default(),
|
||||
input_sources: Default::default(),
|
||||
timeout_seconds: Some(2.0),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
crate::ocr::client::perform(&client, request).await.unwrap();
|
||||
server.await.unwrap();
|
||||
assert_eq!(
|
||||
*names.lock().unwrap(),
|
||||
litellm_llms::mistral::ocr::transformation::MistralOcrConfig.secret_names()
|
||||
);
|
||||
assert!(seen.lock().unwrap()[0].contains("authorization: Bearer source-key"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ocr_client_uses_the_injected_http_pool_configuration() {
|
||||
let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
|
||||
|
|
@ -224,7 +289,7 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() {
|
|||
UrlPolicy::default(),
|
||||
VertexAuth::default(),
|
||||
OcrSettings::default(),
|
||||
Arc::new(litellm_core_utils::settings::ProcessEnvironment),
|
||||
Arc::new(litellm_llms::base_llm::inference::secrets::EnvironmentSecrets),
|
||||
)
|
||||
.unwrap();
|
||||
crate::ocr::client::perform(&client, wire_request("mistral/model", &base, json!({})))
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ pyo3::create_exception!(
|
|||
|
||||
static FORK_GATE: ForkGate = ForkGate::new();
|
||||
|
||||
/// Whether this process has started the Tokio runtime.
|
||||
/// Whether this process has entered process-bound native execution.
|
||||
pub fn runtime_started() -> bool {
|
||||
FORK_GATE.started(std::process::id())
|
||||
}
|
||||
|
|
@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> {
|
|||
FORK_GATE.reserve(std::process::id())
|
||||
}
|
||||
|
||||
/// The only door to the Tokio runtime: every route reaches it through this module, which is
|
||||
/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it.
|
||||
fn enter_runtime() -> PyResult<()> {
|
||||
/// Claims process-bound native state before runtime startup or tokenizer execution.
|
||||
pub fn enter_native() -> PyResult<()> {
|
||||
FORK_GATE
|
||||
.enter(std::process::id())
|
||||
.map_err(|refused| match refused {
|
||||
|
|
@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> {
|
|||
|
||||
#[expect(clippy::disallowed_methods, reason = "this is the gated door")]
|
||||
fn runtime() -> PyResult<&'static Runtime> {
|
||||
enter_runtime()?;
|
||||
enter_native()?;
|
||||
Ok(pyo3_async_runtimes::tokio::get_runtime())
|
||||
}
|
||||
|
||||
|
|
@ -70,7 +69,7 @@ where
|
|||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
T: for<'py> IntoPyObject<'py> + Send + 'static,
|
||||
{
|
||||
enter_runtime()?;
|
||||
enter_native()?;
|
||||
pyo3_async_runtimes::tokio::future_into_py(py, future)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ pub use argument::lookup;
|
|||
pub use callable::wrap_failure;
|
||||
pub use driver::run_call;
|
||||
pub use execution::{
|
||||
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,
|
||||
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value,
|
||||
reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value,
|
||||
runtime_started,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ litellm-auth-gcp.workspace = true
|
|||
litellm-host.workspace = true
|
||||
litellm-framing.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-secrets.workspace = true
|
||||
base64.workspace = true
|
||||
bytes.workspace = true
|
||||
data-url = "0.3.2"
|
||||
|
|
|
|||
|
|
@ -40,6 +40,10 @@ impl BaseOcrConfig for TextractAnalyzeDocumentConfig {
|
|||
type ProviderRequest = AnalyzeDocumentRequest;
|
||||
type Environment = TextractEnvironment;
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
litellm_auth_aws::constants::SECRET_NAMES.to_vec()
|
||||
}
|
||||
|
||||
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
|
||||
&["feature_types"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,10 @@ impl BaseOcrConfig for TextractDetectTextConfig {
|
|||
type ProviderRequest = DetectDocumentTextRequest;
|
||||
type Environment = TextractEnvironment;
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
litellm_auth_aws::constants::SECRET_NAMES.to_vec()
|
||||
}
|
||||
|
||||
fn get_health_check_document(&self) -> OcrDocument {
|
||||
health_check_document()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,10 @@ impl BaseOcrConfig for AzureAICohereParseConfig {
|
|||
super::transformation::AzureAiOcrConfig.get_api_key_env_var()
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
super::transformation::AzureAiOcrConfig.secret_names()
|
||||
}
|
||||
|
||||
fn get_health_check_document(&self) -> OcrDocument {
|
||||
CohereParseConfig.get_health_check_document()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use std::{collections::BTreeSet, time::Duration};
|
|||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use litellm_auth::{InputSource, Sourced};
|
||||
use litellm_auth_azure::AzureAuthInputs;
|
||||
use litellm_auth_azure::{AzureAuthInputs, SECRET_NAMES as AZURE_AUTH_SECRET_NAMES};
|
||||
use litellm_core_utils::{
|
||||
call_arguments::CallArguments,
|
||||
serde_compat::{FiniteF64, LaxI64},
|
||||
|
|
@ -141,6 +141,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
|
|||
Some(AZURE_DI_API_KEY_ENV)
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
[
|
||||
[AZURE_DI_API_KEY_ENV, AZURE_DI_ENDPOINT_ENV].as_slice(),
|
||||
AZURE_AUTH_SECRET_NAMES,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.copied()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials {
|
||||
ResolvedOcrCredentials {
|
||||
api_key: inputs.api_key.and_then(|key| {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use litellm_auth::{InputSource, Sourced};
|
||||
use litellm_auth_azure::AzureAuthInputs;
|
||||
use litellm_auth_azure::SECRET_NAMES as AZURE_AUTH_SECRET_NAMES;
|
||||
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -37,6 +38,17 @@ impl BaseOcrConfig for AzureAiOcrConfig {
|
|||
Some(AZURE_AI_API_KEY_ENV)
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
[
|
||||
[AZURE_AI_API_KEY_ENV, AZURE_AI_API_BASE_ENV].as_slice(),
|
||||
AZURE_AUTH_SECRET_NAMES,
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.copied()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
non_default_params: &CallArguments,
|
||||
|
|
|
|||
1
litellm-rust/crates/llms/src/base_llm/inference/mod.rs
Normal file
1
litellm-rust/crates/llms/src/base_llm/inference/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub mod secrets;
|
||||
19
litellm-rust/crates/llms/src/base_llm/inference/secrets.rs
Normal file
19
litellm-rust/crates/llms/src/base_llm/inference/secrets.rs
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
|
||||
use litellm_secrets::Error;
|
||||
|
||||
pub type Secrets = Arc<dyn Lookup + Send + Sync>;
|
||||
|
||||
pub trait SecretSource: Send + Sync {
|
||||
fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result<Secrets, Error>>;
|
||||
}
|
||||
|
||||
pub struct EnvironmentSecrets;
|
||||
|
||||
impl SecretSource for EnvironmentSecrets {
|
||||
fn resolve<'a>(&'a self, _names: &'a [&'static str]) -> BoxFuture<'a, Result<Secrets, Error>> {
|
||||
Box::pin(async { Ok(Arc::new(ProcessEnvironment) as Secrets) })
|
||||
}
|
||||
}
|
||||
|
|
@ -2,5 +2,6 @@ pub mod anthropic_messages;
|
|||
pub mod audio_transcription;
|
||||
pub mod base_model_iterator;
|
||||
pub mod chat;
|
||||
pub mod inference;
|
||||
pub mod ocr;
|
||||
pub mod responses;
|
||||
|
|
|
|||
|
|
@ -98,6 +98,8 @@ pub enum Error {
|
|||
"Missing REDUCTO_API_KEY - set it in the environment or pass api_key to litellm.ocr()/litellm.aocr()"
|
||||
)]
|
||||
MissingReductoApiKey,
|
||||
#[error("secret resolution failed: {0}")]
|
||||
Secret(#[source] std::sync::Arc<litellm_secrets::Error>),
|
||||
#[error(transparent)]
|
||||
Auth(#[from] litellm_auth::Error),
|
||||
#[error(transparent)]
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::future::BoxFuture;
|
||||
use litellm_auth_gcp::VertexAuth;
|
||||
|
|
@ -11,9 +13,10 @@ use litellm_http::{
|
|||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::base_llm::inference::secrets::SecretSource;
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
settings::{OcrSettings, Secrets},
|
||||
settings::OcrSettings,
|
||||
transformation::{
|
||||
BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext,
|
||||
PreparedOcrRequest, decode_request_value, decode_response,
|
||||
|
|
@ -35,7 +38,7 @@ pub struct OcrClient {
|
|||
document_fetcher: MediaFetcher,
|
||||
vertex_auth: VertexAuth,
|
||||
settings: OcrSettings,
|
||||
secrets: Secrets,
|
||||
secrets: Arc<dyn SecretSource>,
|
||||
}
|
||||
|
||||
impl OcrClient {
|
||||
|
|
@ -45,7 +48,7 @@ impl OcrClient {
|
|||
url_policy: UrlPolicy,
|
||||
vertex_auth: VertexAuth,
|
||||
settings: OcrSettings,
|
||||
secrets: Secrets,
|
||||
secrets: Arc<dyn SecretSource>,
|
||||
) -> Result<Self, litellm_http::Error> {
|
||||
Ok(Self {
|
||||
provider_http: pool.client(config, ClientVariant::Provider)?,
|
||||
|
|
@ -77,7 +80,7 @@ impl OcrClient {
|
|||
&self.settings
|
||||
}
|
||||
|
||||
pub fn secrets(&self) -> &Secrets {
|
||||
pub fn secret_source(&self) -> &Arc<dyn SecretSource> {
|
||||
&self.secrets
|
||||
}
|
||||
|
||||
|
|
@ -92,7 +95,7 @@ impl OcrClient {
|
|||
document_fetcher: MediaFetcher::for_test(document_http),
|
||||
vertex_auth: VertexAuth::default(),
|
||||
settings: OcrSettings::default(),
|
||||
secrets: std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
|
||||
secrets: Arc::new(crate::base_llm::inference::secrets::EnvironmentSecrets),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -102,7 +105,7 @@ impl OcrClient {
|
|||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub fn with_secrets(self, secrets: Secrets) -> Self {
|
||||
pub fn with_secrets(self, secrets: Arc<dyn SecretSource>) -> Self {
|
||||
Self { secrets, ..self }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,7 @@
|
|||
use std::{sync::Arc, time::Duration};
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
|
||||
pub type Secrets = Arc<dyn Lookup + Send + Sync>;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct OcrSettings {
|
||||
pub request_timeout: Duration,
|
||||
|
|
|
|||
|
|
@ -14,10 +14,13 @@ use serde::{
|
|||
use serde_json::{Map, Value};
|
||||
use serde_with::serde_as;
|
||||
|
||||
use crate::base_llm::ocr::{
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body},
|
||||
settings::{OcrSettings, Secrets},
|
||||
use crate::base_llm::{
|
||||
inference::secrets::Secrets,
|
||||
ocr::{
|
||||
error::Error,
|
||||
handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body},
|
||||
settings::OcrSettings,
|
||||
},
|
||||
};
|
||||
|
||||
pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
|
@ -436,6 +439,8 @@ pub trait BaseOcrConfig: Send + Sync + Sized + 'static {
|
|||
None
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str>;
|
||||
|
||||
fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials {
|
||||
ResolvedOcrCredentials {
|
||||
api_key: inputs
|
||||
|
|
|
|||
|
|
@ -102,6 +102,10 @@ impl BaseOcrConfig for CohereParseConfig {
|
|||
Some(COHERE_API_KEY_ENV)
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
vec![COHERE_API_KEY_ENV]
|
||||
}
|
||||
|
||||
fn get_health_check_document(&self) -> OcrDocument {
|
||||
OcrDocument::ImageUrl {
|
||||
image_url: COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI.into(),
|
||||
|
|
|
|||
|
|
@ -69,6 +69,14 @@ impl BaseOcrConfig for MistralOcrConfig {
|
|||
Some(MISTRAL_OCR_API_KEY_ENV_VAR)
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
vec![
|
||||
MISTRAL_OCR_API_KEY_ENV_VAR,
|
||||
"MISTRAL_AZURE_API_KEY",
|
||||
"MISTRAL_AZURE_API_BASE",
|
||||
]
|
||||
}
|
||||
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
non_default_params: &CallArguments,
|
||||
|
|
|
|||
|
|
@ -92,6 +92,10 @@ impl BaseOcrConfig for ReductoParseV3Config {
|
|||
type ProviderRequest = ReductoV3Request;
|
||||
type Environment = Vec<(String, String)>;
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
vec![REDUCTO_API_KEY_ENV]
|
||||
}
|
||||
|
||||
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
|
||||
&["formatting", "retrieval", "settings"]
|
||||
}
|
||||
|
|
@ -180,6 +184,10 @@ impl BaseOcrConfig for ReductoParseLegacyConfig {
|
|||
type ProviderRequest = ReductoLegacyRequest;
|
||||
type Environment = Vec<(String, String)>;
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
vec![REDUCTO_API_KEY_ENV]
|
||||
}
|
||||
|
||||
fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
|
||||
&["enhance"]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,6 +105,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig {
|
|||
VertexAiOcrConfig.get_api_key_env_var()
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
VertexAiOcrConfig.secret_names()
|
||||
}
|
||||
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
_arguments: &CallArguments,
|
||||
|
|
|
|||
|
|
@ -34,6 +34,10 @@ impl BaseOcrConfig for VertexAiOcrConfig {
|
|||
Some("VERTEX_AI_API_KEY")
|
||||
}
|
||||
|
||||
fn secret_names(&self) -> Vec<&'static str> {
|
||||
litellm_auth_gcp::SECRET_NAMES.to_vec()
|
||||
}
|
||||
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
non_default_params: &CallArguments,
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ name = "_native"
|
|||
crate-type = ["cdylib"]
|
||||
|
||||
[features]
|
||||
default = ["abi3", "fast"]
|
||||
default = ["abi3", "fast", "huggingface", "tiktoken"]
|
||||
abi3 = ["pyo3/abi3-py310"]
|
||||
extension-module = ["pyo3/extension-module"]
|
||||
panic-test = []
|
||||
|
|
@ -20,6 +20,7 @@ tiktoken = ["litellm-token-counter/tiktoken"]
|
|||
|
||||
[dependencies]
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-cache-azure-blob.workspace = true
|
||||
litellm-cache-memory.workspace = true
|
||||
|
|
@ -41,6 +42,8 @@ litellm-core-utils.workspace = true
|
|||
litellm-auth-gcp.workspace = true
|
||||
litellm-http.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
litellm-secrets = { workspace = true, features = ["aws"] }
|
||||
litellm-secrets-types.workspace = true
|
||||
litellm-types.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-token-counter = { path = "../token-counter", default-features = false }
|
||||
|
|
@ -53,6 +56,7 @@ url.workspace = true
|
|||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-secrets-aws.workspace = true
|
||||
serde.workspace = true
|
||||
serde_with.workspace = true
|
||||
criterion.workspace = true
|
||||
|
|
@ -60,6 +64,8 @@ futures-util.workspace = true
|
|||
rstest.workspace = true
|
||||
sha2.workspace = true
|
||||
tokio-tungstenite.workspace = true
|
||||
wiremock = "0.6.5"
|
||||
aws-sdk-secretsmanager = "1.117.0"
|
||||
|
||||
[[bench]]
|
||||
name = "serialization"
|
||||
|
|
|
|||
5
litellm-rust/crates/python-bridge/README.md
Normal file
5
litellm-rust/crates/python-bridge/README.md
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
Native OCR uses `SecretSource` with `EnvironmentSecrets`, preserving process-environment reads. Readable Python secret managers still make OCR decline to the existing Python implementation. `ResolvedSecrets` and the separate `secret_manager_binding()` snapshot are inactive foundations for a later rollout
|
||||
|
||||
Cache and secret-manager catalog entries remain Python-only, including when `LITELLM_RUST=1`. The new cache runtime is not connected to SDK or gateway caching
|
||||
|
||||
OCR provider requests use the shared `litellm-http` pool. AWS and Google secret-manager SDK clients keep their SDK transports, which do not yet inherit the pool's proxy, TLS, certificate, timeout, or observability configuration. Preserve those SDK transports and configure them equivalently instead of forcing them through reqwest
|
||||
|
|
@ -1,154 +0,0 @@
|
|||
{
|
||||
"http_settings": {
|
||||
"version": 1,
|
||||
"fields": {
|
||||
"ssl_verify": {
|
||||
"adapter": "SslVerifyInput",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [
|
||||
"none",
|
||||
"bool",
|
||||
"str"
|
||||
],
|
||||
"unsupported_live": "configuration_error"
|
||||
},
|
||||
"ssl_certificate": {
|
||||
"adapter": "OptionalStrictString",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"ssl_security_level": {
|
||||
"adapter": "TuningString",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"ssl_ecdh_curve": {
|
||||
"adapter": "TuningString",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"force_ipv4": {
|
||||
"adapter": "Truthy",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"http2": {
|
||||
"adapter": "ExactTrue",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"aiohttp_trust_env": {
|
||||
"adapter": "Truthy",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"disable_aiohttp_trust_env": {
|
||||
"adapter": "Truthy",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"disable_aiohttp_transport": {
|
||||
"adapter": "ExactTrue",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"user_agent": {
|
||||
"adapter": "StrictString",
|
||||
"required": true,
|
||||
"precedence": "accessor",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"url_policy": {
|
||||
"version": 1,
|
||||
"fields": {
|
||||
"user_url_validation": {
|
||||
"adapter": "Truthy",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"user_url_allowed_hosts": {
|
||||
"adapter": "HostCollection",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"provider_defaults": {
|
||||
"version": 1,
|
||||
"fields": {
|
||||
"vertex_project": {
|
||||
"adapter": "FalsyOptionalString",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": true,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"vertex_location": {
|
||||
"adapter": "FalsyOptionalString",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": true,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
},
|
||||
"enable_azure_ad_token_refresh": {
|
||||
"adapter": "ExactTrue",
|
||||
"required": true,
|
||||
"precedence": "module_global",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
}
|
||||
}
|
||||
},
|
||||
"secret_manager": {
|
||||
"version": 1,
|
||||
"fields": {
|
||||
"readable": {
|
||||
"adapter": "StrictBool",
|
||||
"required": true,
|
||||
"precedence": "accessor",
|
||||
"sensitive": false,
|
||||
"shapes": [],
|
||||
"unsupported_live": null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -11,10 +11,12 @@ use serde_json::Value;
|
|||
use super::{
|
||||
cache_error,
|
||||
callback::PythonCallback,
|
||||
config::{CacheBackendConfig, CacheConfigProjection, NativeCacheConfig},
|
||||
future::{ready_none, ready_value},
|
||||
native::NativeResponseCache,
|
||||
request::{now, request, requests},
|
||||
};
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
pub(super) enum CacheBinding {
|
||||
Disabled,
|
||||
|
|
@ -22,7 +24,7 @@ pub(super) enum CacheBinding {
|
|||
PythonCallback(PythonCallback),
|
||||
}
|
||||
|
||||
#[pyclass(frozen, name = "_CacheTestBinding")]
|
||||
#[pyclass(frozen, name = "_ResponseCacheRuntime")]
|
||||
pub(crate) struct ResolvedCache {
|
||||
binding: CacheBinding,
|
||||
pid: u32,
|
||||
|
|
@ -66,6 +68,33 @@ impl ResolvedCache {
|
|||
|
||||
#[pymethods]
|
||||
impl ResolvedCache {
|
||||
#[staticmethod]
|
||||
fn from_cache(cache: &Bound<'_, PyAny>) -> PyResult<Self> {
|
||||
let config = match NativeCacheConfig::project(cache)? {
|
||||
CacheConfigProjection::Native(config) => *config,
|
||||
CacheConfigProjection::Unsupported(reason) => {
|
||||
return Err(RustBridgeDeclined::new_err(reason.message()));
|
||||
}
|
||||
};
|
||||
let service = match config.backend {
|
||||
CacheBackendConfig::Memory(memory) => NativeResponseCache::memory(
|
||||
memory.capacity,
|
||||
memory.default_ttl,
|
||||
memory.max_entry_bytes,
|
||||
),
|
||||
_ => {
|
||||
return Err(RustBridgeDeclined::new_err(
|
||||
"native response cache activation is not implemented for this backend",
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok(Self::new(CacheBinding::Native(
|
||||
service
|
||||
.with_scope(config.policy.semantic_cache_scope)
|
||||
.with_redis_flush_size(config.policy.redis_flush_size),
|
||||
)))
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn kind(&self) -> &'static str {
|
||||
match self.binding {
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use pyo3::{
|
|||
types::{PyAny, PyBool, PyDict, PyList, PyString},
|
||||
};
|
||||
|
||||
use super::{native::NativeResponseCache, request::duration};
|
||||
use super::{identity::BackendIdentity, native::NativeResponseCache, request::duration};
|
||||
|
||||
#[allow(dead_code, reason = "consumed by the cache activation follow-up")]
|
||||
pub(super) struct CachePolicy {
|
||||
|
|
@ -293,161 +293,58 @@ impl NativeCacheConfig {
|
|||
}
|
||||
|
||||
pub(super) fn service_mismatch(&self, service: &NativeResponseCache) -> Option<&'static str> {
|
||||
let default_ttl = match &self.backend {
|
||||
CacheBackendConfig::Memory(config) => Some(config.default_ttl),
|
||||
CacheBackendConfig::Redis(config) => Some(config.default_ttl),
|
||||
CacheBackendConfig::S3(_) => None,
|
||||
CacheBackendConfig::ValkeySemantic(_) => Some(Duration::ZERO),
|
||||
CacheBackendConfig::Disk(_)
|
||||
| CacheBackendConfig::AzureBlob(_)
|
||||
| CacheBackendConfig::Gcs(_)
|
||||
| CacheBackendConfig::RedisSemantic(_)
|
||||
| CacheBackendConfig::QdrantSemantic(_) => None,
|
||||
};
|
||||
if !matches!(self.backend, CacheBackendConfig::ValkeySemantic(_))
|
||||
&& service.default_ttl() != default_ttl
|
||||
{
|
||||
return Some("facade and native backend default TTLs must match");
|
||||
}
|
||||
match &self.backend {
|
||||
CacheBackendConfig::Memory(config) if service.kind() != "memory" => {
|
||||
Some("facade and native backend types must match")
|
||||
}
|
||||
CacheBackendConfig::Memory(config) if service.capacity() != Some(config.capacity) => {
|
||||
Some("facade and native backend capacities must match")
|
||||
}
|
||||
CacheBackendConfig::Memory(config)
|
||||
if service.max_entry_bytes() != Some(config.max_entry_bytes) =>
|
||||
{
|
||||
Some("facade and native backend item limits must match")
|
||||
}
|
||||
CacheBackendConfig::Memory(_) => None,
|
||||
CacheBackendConfig::Redis(_) if service.kind() != "redis" => {
|
||||
Some("facade and native backend types must match")
|
||||
}
|
||||
CacheBackendConfig::Redis(config) if service.topology() != Some(&config.topology) => {
|
||||
Some("facade and native backend topologies must match")
|
||||
}
|
||||
CacheBackendConfig::Redis(config) => (service.namespace()
|
||||
!= config.namespace.as_deref())
|
||||
.then_some("facade and native backend namespaces must match"),
|
||||
CacheBackendConfig::S3(_) if service.kind() != "s3" => {
|
||||
Some("facade and native backend types must match")
|
||||
}
|
||||
CacheBackendConfig::S3(config) if service.bucket() != Some(config.bucket.as_str()) => {
|
||||
Some("facade and native backend buckets must match")
|
||||
}
|
||||
CacheBackendConfig::S3(config)
|
||||
if service.key_prefix() != Some(config.key_prefix.as_str()) =>
|
||||
{
|
||||
Some("facade and native backend key prefixes must match")
|
||||
}
|
||||
CacheBackendConfig::S3(config) if service.region() != Some(config.region.as_str()) => {
|
||||
Some("facade and native backend regions must match")
|
||||
}
|
||||
CacheBackendConfig::S3(config)
|
||||
if service.endpoint()
|
||||
!= config
|
||||
.endpoint
|
||||
.as_ref()
|
||||
.map(|endpoint| endpoint.url.as_str()) =>
|
||||
{
|
||||
Some("facade and native backend endpoints must match")
|
||||
}
|
||||
CacheBackendConfig::S3(_) => None,
|
||||
CacheBackendConfig::Gcs(_) if service.kind() != "gcs" => {
|
||||
Some("facade and native backend types must match")
|
||||
}
|
||||
CacheBackendConfig::Gcs(config)
|
||||
if service
|
||||
.gcs_backend()
|
||||
.is_none_or(|backend| backend.bucket_name() != config.bucket_name) =>
|
||||
{
|
||||
Some("facade and native backend buckets must match")
|
||||
}
|
||||
CacheBackendConfig::Gcs(config)
|
||||
if service
|
||||
.gcs_backend()
|
||||
.is_none_or(|backend| backend.key_prefix() != config.key_prefix) =>
|
||||
{
|
||||
Some("facade and native backend key prefixes must match")
|
||||
}
|
||||
CacheBackendConfig::Gcs(config)
|
||||
if service.gcs_backend().is_none_or(|backend| {
|
||||
backend.path_service_account() != config.path_service_account.as_deref()
|
||||
}) =>
|
||||
{
|
||||
Some("facade and native backend credentials must match")
|
||||
}
|
||||
CacheBackendConfig::Gcs(_) => None,
|
||||
CacheBackendConfig::ValkeySemantic(config) => {
|
||||
if service.kind() != "valkey-semantic" {
|
||||
return Some("facade and native backend types must match");
|
||||
}
|
||||
let Some((threshold, index_name)) = service.semantic_config() else {
|
||||
return Some("facade and native backend types must match");
|
||||
};
|
||||
(threshold != config.similarity_threshold || index_name != config.index_name)
|
||||
.then_some("facade and native semantic settings must match")
|
||||
}
|
||||
CacheBackendConfig::Disk(_) if service.kind() != "disk" => {
|
||||
Some("facade and native backend types must match")
|
||||
}
|
||||
CacheBackendConfig::Disk(config) => {
|
||||
let Some(directory) = service.directory() else {
|
||||
return Some("facade and native backend types must match");
|
||||
};
|
||||
let native = std::fs::canonicalize(directory).ok();
|
||||
let facade = std::fs::canonicalize(&config.directory).ok();
|
||||
(native != facade).then_some("facade and native backend directories must match")
|
||||
}
|
||||
CacheBackendConfig::RedisSemantic(_) if service.kind() != "redis_semantic" => {
|
||||
Some("facade and native backend types must match")
|
||||
}
|
||||
CacheBackendConfig::RedisSemantic(config)
|
||||
if service.index_name() != Some(config.index_name.as_str()) =>
|
||||
{
|
||||
Some("facade and native backend index names must match")
|
||||
}
|
||||
CacheBackendConfig::RedisSemantic(config)
|
||||
if service.similarity_threshold()
|
||||
!= Some(f64::from(config.similarity_threshold as f32)) =>
|
||||
{
|
||||
Some("facade and native backend similarity thresholds must match")
|
||||
}
|
||||
CacheBackendConfig::RedisSemantic(_) => None,
|
||||
CacheBackendConfig::QdrantSemantic(config) if service.kind() != "qdrant_semantic" => {
|
||||
Some("facade and native backend types must match")
|
||||
}
|
||||
CacheBackendConfig::QdrantSemantic(config)
|
||||
if service.collection_name() != Some(config.collection_name.as_str()) =>
|
||||
{
|
||||
Some("facade and native backend collections must match")
|
||||
}
|
||||
CacheBackendConfig::QdrantSemantic(config)
|
||||
if service.similarity_threshold() != Some(config.similarity_threshold) =>
|
||||
{
|
||||
Some("facade and native backend similarity thresholds must match")
|
||||
}
|
||||
CacheBackendConfig::QdrantSemantic(config)
|
||||
if service.vector_size() != Some(config.vector_size) =>
|
||||
{
|
||||
Some("facade and native backend vector sizes must match")
|
||||
}
|
||||
CacheBackendConfig::QdrantSemantic(config)
|
||||
if service.embedding_model() != Some(config.embedding.model.as_str()) =>
|
||||
{
|
||||
Some("facade and native backend embedding models must match")
|
||||
}
|
||||
CacheBackendConfig::QdrantSemantic(_) => None,
|
||||
CacheBackendConfig::AzureBlob(config) => match service.azure_blob_identity() {
|
||||
None => Some("facade and native backend types must match"),
|
||||
Some((account_url, container))
|
||||
if account_url != config.account_url || container != config.container =>
|
||||
{
|
||||
Some("facade and native backend containers must match")
|
||||
}
|
||||
Some(_) => None,
|
||||
self.backend.identity().mismatch(&service.identity())
|
||||
}
|
||||
}
|
||||
|
||||
impl CacheBackendConfig {
|
||||
/// The identity a native backend must have for this facade configuration to describe it.
|
||||
pub(super) fn identity(&self) -> BackendIdentity {
|
||||
match self {
|
||||
Self::Memory(config) => BackendIdentity::Memory {
|
||||
capacity: config.capacity,
|
||||
max_entry_bytes: Some(config.max_entry_bytes),
|
||||
default_ttl: Some(config.default_ttl),
|
||||
},
|
||||
Self::Redis(config) => BackendIdentity::Redis {
|
||||
topology: config.topology.clone(),
|
||||
namespace: config.namespace.clone(),
|
||||
default_ttl: Some(config.default_ttl),
|
||||
},
|
||||
Self::S3(config) => BackendIdentity::S3 {
|
||||
bucket: config.bucket.clone(),
|
||||
key_prefix: config.key_prefix.clone(),
|
||||
region: config.region.clone(),
|
||||
endpoint: config
|
||||
.endpoint
|
||||
.as_ref()
|
||||
.map(|endpoint| endpoint.url.clone()),
|
||||
},
|
||||
Self::Gcs(config) => BackendIdentity::Gcs {
|
||||
bucket_name: config.bucket_name.clone(),
|
||||
key_prefix: config.key_prefix.clone(),
|
||||
path_service_account: config.path_service_account.clone(),
|
||||
},
|
||||
Self::ValkeySemantic(config) => BackendIdentity::ValkeySemantic {
|
||||
index_name: config.index_name.clone(),
|
||||
similarity_threshold: config.similarity_threshold,
|
||||
},
|
||||
Self::Disk(config) => BackendIdentity::Disk {
|
||||
directory: config.directory.clone(),
|
||||
},
|
||||
Self::AzureBlob(config) => BackendIdentity::AzureBlob {
|
||||
account_url: config.account_url.clone(),
|
||||
container: config.container.clone(),
|
||||
},
|
||||
Self::RedisSemantic(config) => BackendIdentity::RedisSemantic {
|
||||
index_name: config.index_name.clone(),
|
||||
similarity_threshold: config.similarity_threshold as f32,
|
||||
},
|
||||
Self::QdrantSemantic(config) => BackendIdentity::QdrantSemantic {
|
||||
collection_name: config.collection_name.clone(),
|
||||
similarity_threshold: config.similarity_threshold,
|
||||
vector_size: config.vector_size,
|
||||
embedding_model: config.embedding.model.clone(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ tokio::task_local! {
|
|||
static PREPARED_EMBEDDING: Result<Vec<f32>, Error>;
|
||||
}
|
||||
|
||||
/// Runs `future` with the vector the Python embedder already produced, so the backend's
|
||||
/// `async_embed` never has to call back into Python from the runtime.
|
||||
pub(super) fn with_prepared_embedding<F: Future>(
|
||||
vector: Result<Vec<f32>, Error>,
|
||||
future: F,
|
||||
|
|
@ -16,6 +18,7 @@ pub(super) fn with_prepared_embedding<F: Future>(
|
|||
PREPARED_EMBEDDING.scope(vector, future)
|
||||
}
|
||||
|
||||
/// The Python object that owns embedding for a semantic backend.
|
||||
pub(super) struct PythonEmbedder(Py<PyAny>);
|
||||
|
||||
impl Clone for PythonEmbedder {
|
||||
|
|
@ -29,10 +32,6 @@ impl PythonEmbedder {
|
|||
Self(object)
|
||||
}
|
||||
|
||||
pub(super) fn from_backend(backend: &Bound<'_, PyAny>) -> PyResult<Self> {
|
||||
Ok(Self(backend.clone().unbind()))
|
||||
}
|
||||
|
||||
pub(super) fn object(&self) -> &Py<PyAny> {
|
||||
&self.0
|
||||
}
|
||||
|
|
@ -41,18 +40,6 @@ impl PythonEmbedder {
|
|||
visit.call(&self.0)
|
||||
}
|
||||
|
||||
pub(super) fn async_embed_awaitable<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
prompt: &str,
|
||||
metadata: &Option<Value>,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let metadata = to_py(py, metadata)?;
|
||||
self.0
|
||||
.bind(py)
|
||||
.call_method1("_get_async_embedding", (prompt, metadata))
|
||||
}
|
||||
|
||||
fn metadata_kwargs<'py>(
|
||||
py: Python<'py>,
|
||||
metadata: Option<&Value>,
|
||||
|
|
@ -62,7 +49,8 @@ impl PythonEmbedder {
|
|||
Ok(kwargs)
|
||||
}
|
||||
|
||||
pub(super) fn async_embedding_coroutine(
|
||||
/// The awaitable of `_get_async_embedding(prompt, metadata=...)`, to run in the caller's loop.
|
||||
pub(super) fn async_embedding(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
prompt: &str,
|
||||
|
|
@ -82,35 +70,8 @@ impl PythonEmbedder {
|
|||
.map(|value| value as f32)
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
|
||||
impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder {
|
||||
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
|
||||
let result = Python::attach(|py| -> PyResult<Vec<f64>> {
|
||||
let metadata = to_py(py, &metadata)?;
|
||||
self.0
|
||||
.bind(py)
|
||||
.call_method1("_get_embedding", (prompt, metadata))?
|
||||
.extract()
|
||||
})
|
||||
.map_err(|_| Error::Unavailable)?;
|
||||
Ok(result.into_iter().map(|value| value as f32).collect())
|
||||
}
|
||||
|
||||
fn async_embed(
|
||||
&self,
|
||||
_prompt: &str,
|
||||
_metadata: Option<&Value>,
|
||||
) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
|
||||
let seeded = PREPARED_EMBEDDING
|
||||
.try_with(Clone::clone)
|
||||
.unwrap_or(Err(Error::Unavailable));
|
||||
std::future::ready(seeded)
|
||||
}
|
||||
}
|
||||
|
||||
impl litellm_cache_redis_semantic::Embedder for PythonEmbedder {
|
||||
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
|
||||
fn embed_sync(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
|
||||
Python::attach(|py| {
|
||||
let kwargs = Self::metadata_kwargs(py, metadata)?;
|
||||
Self::extract(self.0.bind(py).call_method(
|
||||
|
|
@ -122,15 +83,38 @@ impl litellm_cache_redis_semantic::Embedder for PythonEmbedder {
|
|||
.map_err(|_| Error::Unavailable)
|
||||
}
|
||||
|
||||
fn seeded_embedding() -> Result<Vec<f32>, Error> {
|
||||
PREPARED_EMBEDDING
|
||||
.try_with(Clone::clone)
|
||||
.unwrap_or(Err(Error::Unavailable))
|
||||
}
|
||||
}
|
||||
|
||||
impl litellm_cache_valkey_semantic::Embedder for PythonEmbedder {
|
||||
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
|
||||
self.embed_sync(prompt, metadata)
|
||||
}
|
||||
|
||||
fn async_embed(
|
||||
&self,
|
||||
_prompt: &str,
|
||||
_metadata: Option<&Value>,
|
||||
) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
|
||||
let seeded = PREPARED_EMBEDDING
|
||||
.try_with(Clone::clone)
|
||||
.unwrap_or(Err(Error::Unavailable));
|
||||
std::future::ready(seeded)
|
||||
std::future::ready(Self::seeded_embedding())
|
||||
}
|
||||
}
|
||||
|
||||
impl litellm_cache_redis_semantic::Embedder for PythonEmbedder {
|
||||
fn embed(&self, prompt: &str, metadata: Option<&Value>) -> Result<Vec<f32>, Error> {
|
||||
self.embed_sync(prompt, metadata)
|
||||
}
|
||||
|
||||
fn async_embed(
|
||||
&self,
|
||||
_prompt: &str,
|
||||
_metadata: Option<&Value>,
|
||||
) -> impl Future<Output = Result<Vec<f32>, Error>> + Send {
|
||||
std::future::ready(Self::seeded_embedding())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -152,5 +136,9 @@ mod tests {
|
|||
let unscoped =
|
||||
litellm_cache_redis_semantic::Embedder::async_embed(&embedder, "prompt", None).await;
|
||||
assert_eq!(unscoped, Err(Error::Unavailable));
|
||||
let valkey = with_prepared_embedding(Ok(vec![0.5]), async move {
|
||||
litellm_cache_valkey_semantic::Embedder::async_embed(&embedder, "prompt", None).await
|
||||
});
|
||||
assert_eq!(valkey.await, Ok(vec![0.5]));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ use serde_json::Value;
|
|||
use super::{
|
||||
config::{CacheConfigProjection, NativeCacheConfig},
|
||||
handle::CacheTestHandle,
|
||||
identity::BackendIdentity,
|
||||
native::NativeResponseCache,
|
||||
};
|
||||
|
||||
|
|
@ -352,47 +353,41 @@ impl FacadeGuard {
|
|||
facade: &Bound<'_, PyAny>,
|
||||
service: &NativeResponseCache,
|
||||
) -> PyResult<Self> {
|
||||
let kind = service.kind();
|
||||
let identity = service.identity();
|
||||
let kind = identity.kind();
|
||||
let cache_type = py.import("litellm.caching.caching")?.getattr("Cache")?;
|
||||
if !facade.get_type().is(&cache_type) {
|
||||
return Err(PyTypeError::new_err(
|
||||
"only exact built-in Cache facades can be registered",
|
||||
));
|
||||
}
|
||||
let cluster = matches!(service.topology(), Some(RedisTopology::Cluster { .. }));
|
||||
let (module, name, cache_kind) = match (kind, cluster) {
|
||||
("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache", "local"),
|
||||
("redis", false) => ("litellm.caching.redis_cache", "RedisCache", "redis"),
|
||||
("redis_semantic", _) => (
|
||||
"litellm.caching.redis_semantic_cache",
|
||||
"RedisSemanticCache",
|
||||
"redis-semantic",
|
||||
),
|
||||
let cluster = matches!(
|
||||
identity,
|
||||
BackendIdentity::Redis {
|
||||
topology: RedisTopology::Cluster { .. },
|
||||
..
|
||||
}
|
||||
);
|
||||
let (module, name) = match (kind, cluster) {
|
||||
("memory", _) => ("litellm.caching.in_memory_cache", "InMemoryCache"),
|
||||
("redis", false) => ("litellm.caching.redis_cache", "RedisCache"),
|
||||
("redis", true) => ("litellm.caching.redis_cluster_cache", "RedisClusterCache"),
|
||||
("redis_semantic", _) => ("litellm.caching.redis_semantic_cache", "RedisSemanticCache"),
|
||||
("qdrant_semantic", _) => (
|
||||
"litellm.caching.qdrant_semantic_cache",
|
||||
"QdrantSemanticCache",
|
||||
"qdrant-semantic",
|
||||
),
|
||||
("redis", true) => (
|
||||
"litellm.caching.redis_cluster_cache",
|
||||
"RedisClusterCache",
|
||||
"redis",
|
||||
),
|
||||
("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache", "gcs"),
|
||||
("valkey-semantic", false) => (
|
||||
("gcs", _) => ("litellm.caching.gcs_cache", "GCSCache"),
|
||||
("valkey-semantic", _) => (
|
||||
"litellm.caching.valkey_semantic_cache",
|
||||
"ValkeySemanticCache",
|
||||
"valkey-semantic",
|
||||
),
|
||||
("disk", _) => ("litellm.caching.disk_cache", "DiskCache", "disk"),
|
||||
("azure-blob", _) => (
|
||||
"litellm.caching.azure_blob_cache",
|
||||
"AzureBlobCache",
|
||||
"azure-blob",
|
||||
),
|
||||
("s3", _) => ("litellm.caching.s3_cache", "S3Cache", "s3"),
|
||||
("disk", _) => ("litellm.caching.disk_cache", "DiskCache"),
|
||||
("azure-blob", _) => ("litellm.caching.azure_blob_cache", "AzureBlobCache"),
|
||||
("s3", _) => ("litellm.caching.s3_cache", "S3Cache"),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let cache_kind = identity.cache_type();
|
||||
let backend = facade.getattr("cache")?;
|
||||
if facade.getattr("type")?.extract::<String>()? != cache_kind
|
||||
|| !backend.get_type().is(&py.import(module)?.getattr(name)?)
|
||||
|
|
|
|||
|
|
@ -261,7 +261,7 @@ impl CacheTestHandle {
|
|||
index_name: String,
|
||||
embedder: &Bound<'_, PyAny>,
|
||||
) -> PyResult<Self> {
|
||||
let python_embedder = PythonEmbedder::from_backend(embedder)?;
|
||||
let python_embedder = PythonEmbedder::new(embedder.clone().unbind());
|
||||
let service = NativeResponseCache::valkey_semantic(
|
||||
&url,
|
||||
similarity_threshold,
|
||||
|
|
|
|||
511
litellm-rust/crates/python-bridge/src/cache/identity.rs
vendored
Normal file
511
litellm-rust/crates/python-bridge/src/cache/identity.rs
vendored
Normal file
|
|
@ -0,0 +1,511 @@
|
|||
use std::{path::PathBuf, time::Duration};
|
||||
|
||||
use litellm_cache_redis::RedisTopology;
|
||||
|
||||
/// What makes a native backend the one a Python facade describes: the configuration a user can
|
||||
/// observe on the Python object, captured once so facade projection and native construction
|
||||
/// compare plain data instead of reaching into each backend type.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub(super) enum BackendIdentity {
|
||||
Memory {
|
||||
capacity: usize,
|
||||
max_entry_bytes: Option<usize>,
|
||||
default_ttl: Option<Duration>,
|
||||
},
|
||||
Redis {
|
||||
topology: RedisTopology,
|
||||
namespace: Option<String>,
|
||||
default_ttl: Option<Duration>,
|
||||
},
|
||||
S3 {
|
||||
bucket: String,
|
||||
key_prefix: String,
|
||||
region: String,
|
||||
endpoint: Option<String>,
|
||||
},
|
||||
Gcs {
|
||||
bucket_name: String,
|
||||
key_prefix: String,
|
||||
path_service_account: Option<String>,
|
||||
},
|
||||
Disk {
|
||||
directory: PathBuf,
|
||||
},
|
||||
AzureBlob {
|
||||
account_url: String,
|
||||
container: String,
|
||||
},
|
||||
RedisSemantic {
|
||||
index_name: String,
|
||||
/// The backend stores the threshold as `f32`; a facade's `f64` is compared at that width.
|
||||
similarity_threshold: f32,
|
||||
},
|
||||
ValkeySemantic {
|
||||
index_name: String,
|
||||
similarity_threshold: f64,
|
||||
},
|
||||
QdrantSemantic {
|
||||
collection_name: String,
|
||||
similarity_threshold: f64,
|
||||
vector_size: u64,
|
||||
embedding_model: String,
|
||||
},
|
||||
}
|
||||
|
||||
const TYPES: &str = "facade and native backend types must match";
|
||||
|
||||
impl BackendIdentity {
|
||||
/// The native backend name reported to Python through `_CacheTestHandle.backend`.
|
||||
pub(super) fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Memory { .. } => "memory",
|
||||
Self::Redis { .. } => "redis",
|
||||
Self::S3 { .. } => "s3",
|
||||
Self::Gcs { .. } => "gcs",
|
||||
Self::ValkeySemantic { .. } => "valkey-semantic",
|
||||
Self::RedisSemantic { .. } => "redis_semantic",
|
||||
Self::QdrantSemantic { .. } => "qdrant_semantic",
|
||||
Self::Disk { .. } => "disk",
|
||||
Self::AzureBlob { .. } => "azure-blob",
|
||||
}
|
||||
}
|
||||
|
||||
/// The `LiteLLMCacheType` value a facade of this backend carries in `Cache.type`.
|
||||
pub(super) fn cache_type(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Memory { .. } => "local",
|
||||
Self::Redis { .. } => "redis",
|
||||
Self::S3 { .. } => "s3",
|
||||
Self::Gcs { .. } => "gcs",
|
||||
Self::ValkeySemantic { .. } => "valkey-semantic",
|
||||
Self::RedisSemantic { .. } => "redis-semantic",
|
||||
Self::QdrantSemantic { .. } => "qdrant-semantic",
|
||||
Self::Disk { .. } => "disk",
|
||||
Self::AzureBlob { .. } => "azure-blob",
|
||||
}
|
||||
}
|
||||
|
||||
/// The first difference between the facade's configuration (`self`) and the native
|
||||
/// backend (`native`), in the order Python users see the attributes.
|
||||
pub(super) fn mismatch(&self, native: &Self) -> Option<&'static str> {
|
||||
let mut differences: Vec<(bool, &'static str)> = Vec::new();
|
||||
let mut differs = |condition: bool, message: &'static str| {
|
||||
differences.push((condition, message));
|
||||
};
|
||||
match (self, native) {
|
||||
(
|
||||
Self::Memory {
|
||||
capacity,
|
||||
max_entry_bytes,
|
||||
default_ttl,
|
||||
},
|
||||
Self::Memory {
|
||||
capacity: native_capacity,
|
||||
max_entry_bytes: native_max_entry_bytes,
|
||||
default_ttl: native_default_ttl,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
default_ttl != native_default_ttl,
|
||||
"facade and native backend default TTLs must match",
|
||||
);
|
||||
differs(
|
||||
capacity != native_capacity,
|
||||
"facade and native backend capacities must match",
|
||||
);
|
||||
differs(
|
||||
max_entry_bytes != native_max_entry_bytes,
|
||||
"facade and native backend item limits must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::Redis {
|
||||
topology,
|
||||
namespace,
|
||||
default_ttl,
|
||||
},
|
||||
Self::Redis {
|
||||
topology: native_topology,
|
||||
namespace: native_namespace,
|
||||
default_ttl: native_default_ttl,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
default_ttl != native_default_ttl,
|
||||
"facade and native backend default TTLs must match",
|
||||
);
|
||||
differs(
|
||||
topology != native_topology,
|
||||
"facade and native backend topologies must match",
|
||||
);
|
||||
differs(
|
||||
namespace != native_namespace,
|
||||
"facade and native backend namespaces must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::S3 {
|
||||
bucket,
|
||||
key_prefix,
|
||||
region,
|
||||
endpoint,
|
||||
},
|
||||
Self::S3 {
|
||||
bucket: native_bucket,
|
||||
key_prefix: native_key_prefix,
|
||||
region: native_region,
|
||||
endpoint: native_endpoint,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
bucket != native_bucket,
|
||||
"facade and native backend buckets must match",
|
||||
);
|
||||
differs(
|
||||
key_prefix != native_key_prefix,
|
||||
"facade and native backend key prefixes must match",
|
||||
);
|
||||
differs(
|
||||
region != native_region,
|
||||
"facade and native backend regions must match",
|
||||
);
|
||||
differs(
|
||||
endpoint != native_endpoint,
|
||||
"facade and native backend endpoints must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::Gcs {
|
||||
bucket_name,
|
||||
key_prefix,
|
||||
path_service_account,
|
||||
},
|
||||
Self::Gcs {
|
||||
bucket_name: native_bucket_name,
|
||||
key_prefix: native_key_prefix,
|
||||
path_service_account: native_path_service_account,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
bucket_name != native_bucket_name,
|
||||
"facade and native backend buckets must match",
|
||||
);
|
||||
differs(
|
||||
key_prefix != native_key_prefix,
|
||||
"facade and native backend key prefixes must match",
|
||||
);
|
||||
differs(
|
||||
path_service_account != native_path_service_account,
|
||||
"facade and native backend credentials must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::Disk { directory },
|
||||
Self::Disk {
|
||||
directory: native_directory,
|
||||
},
|
||||
) => {
|
||||
let canonical = |path: &PathBuf| std::fs::canonicalize(path).ok();
|
||||
differs(
|
||||
canonical(directory) != canonical(native_directory),
|
||||
"facade and native backend directories must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::AzureBlob {
|
||||
account_url,
|
||||
container,
|
||||
},
|
||||
Self::AzureBlob {
|
||||
account_url: native_account_url,
|
||||
container: native_container,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
account_url != native_account_url || container != native_container,
|
||||
"facade and native backend containers must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::RedisSemantic {
|
||||
index_name,
|
||||
similarity_threshold,
|
||||
},
|
||||
Self::RedisSemantic {
|
||||
index_name: native_index_name,
|
||||
similarity_threshold: native_similarity_threshold,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
index_name != native_index_name,
|
||||
"facade and native backend index names must match",
|
||||
);
|
||||
differs(
|
||||
similarity_threshold != native_similarity_threshold,
|
||||
"facade and native backend similarity thresholds must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::ValkeySemantic {
|
||||
index_name,
|
||||
similarity_threshold,
|
||||
},
|
||||
Self::ValkeySemantic {
|
||||
index_name: native_index_name,
|
||||
similarity_threshold: native_similarity_threshold,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
index_name != native_index_name
|
||||
|| similarity_threshold != native_similarity_threshold,
|
||||
"facade and native semantic settings must match",
|
||||
);
|
||||
}
|
||||
(
|
||||
Self::QdrantSemantic {
|
||||
collection_name,
|
||||
similarity_threshold,
|
||||
vector_size,
|
||||
embedding_model,
|
||||
},
|
||||
Self::QdrantSemantic {
|
||||
collection_name: native_collection_name,
|
||||
similarity_threshold: native_similarity_threshold,
|
||||
vector_size: native_vector_size,
|
||||
embedding_model: native_embedding_model,
|
||||
},
|
||||
) => {
|
||||
differs(
|
||||
collection_name != native_collection_name,
|
||||
"facade and native backend collections must match",
|
||||
);
|
||||
differs(
|
||||
similarity_threshold != native_similarity_threshold,
|
||||
"facade and native backend similarity thresholds must match",
|
||||
);
|
||||
differs(
|
||||
vector_size != native_vector_size,
|
||||
"facade and native backend vector sizes must match",
|
||||
);
|
||||
differs(
|
||||
embedding_model != native_embedding_model,
|
||||
"facade and native backend embedding models must match",
|
||||
);
|
||||
}
|
||||
_ => return Some(TYPES),
|
||||
}
|
||||
differences
|
||||
.into_iter()
|
||||
.find_map(|(condition, message)| condition.then_some(message))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::Duration;
|
||||
|
||||
use litellm_cache_redis::{RedisNode, RedisTopology};
|
||||
|
||||
use super::BackendIdentity;
|
||||
|
||||
fn memory() -> BackendIdentity {
|
||||
BackendIdentity::Memory {
|
||||
capacity: 200,
|
||||
max_entry_bytes: Some(1024),
|
||||
default_ttl: Some(Duration::from_secs(60)),
|
||||
}
|
||||
}
|
||||
|
||||
fn redis() -> BackendIdentity {
|
||||
BackendIdentity::Redis {
|
||||
topology: RedisTopology::Standalone,
|
||||
namespace: Some("team".into()),
|
||||
default_ttl: Some(Duration::from_secs(60)),
|
||||
}
|
||||
}
|
||||
|
||||
fn s3() -> BackendIdentity {
|
||||
BackendIdentity::S3 {
|
||||
bucket: "bucket".into(),
|
||||
key_prefix: "cache/".into(),
|
||||
region: "us-east-1".into(),
|
||||
endpoint: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn gcs() -> BackendIdentity {
|
||||
BackendIdentity::Gcs {
|
||||
bucket_name: "bucket".into(),
|
||||
key_prefix: "cache/".into(),
|
||||
path_service_account: Some("credentials.json".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn azure() -> BackendIdentity {
|
||||
BackendIdentity::AzureBlob {
|
||||
account_url: "https://account.blob.core.windows.net".into(),
|
||||
container: "cache".into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn redis_semantic() -> BackendIdentity {
|
||||
BackendIdentity::RedisSemantic {
|
||||
index_name: "idx".into(),
|
||||
similarity_threshold: 0.8,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redis_semantic_thresholds_compare_at_backend_precision() {
|
||||
let facade = BackendIdentity::RedisSemantic {
|
||||
index_name: "idx".into(),
|
||||
similarity_threshold: 0.8_f64 as f32,
|
||||
};
|
||||
assert_eq!(facade.mismatch(&redis_semantic()), None);
|
||||
}
|
||||
|
||||
fn valkey_semantic() -> BackendIdentity {
|
||||
BackendIdentity::ValkeySemantic {
|
||||
index_name: "idx".into(),
|
||||
similarity_threshold: 0.8,
|
||||
}
|
||||
}
|
||||
|
||||
fn qdrant() -> BackendIdentity {
|
||||
BackendIdentity::QdrantSemantic {
|
||||
collection_name: "collection".into(),
|
||||
similarity_threshold: 0.8,
|
||||
vector_size: 1536,
|
||||
embedding_model: "text-embedding-3-small".into(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_identities_have_no_mismatch() {
|
||||
for identity in [
|
||||
memory(),
|
||||
redis(),
|
||||
s3(),
|
||||
gcs(),
|
||||
azure(),
|
||||
redis_semantic(),
|
||||
valkey_semantic(),
|
||||
qdrant(),
|
||||
BackendIdentity::Disk {
|
||||
directory: std::env::temp_dir(),
|
||||
},
|
||||
] {
|
||||
assert_eq!(identity.mismatch(&identity), None, "{identity:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_kinds_report_a_type_mismatch() {
|
||||
assert_eq!(
|
||||
memory().mismatch(&redis()),
|
||||
Some("facade and native backend types must match")
|
||||
);
|
||||
assert_eq!(
|
||||
redis_semantic().mismatch(&valkey_semantic()),
|
||||
Some("facade and native backend types must match")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_first_differing_field_names_the_mismatch() {
|
||||
let BackendIdentity::Memory { capacity, .. } = memory() else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(
|
||||
memory().mismatch(&BackendIdentity::Memory {
|
||||
capacity: capacity + 1,
|
||||
max_entry_bytes: Some(1),
|
||||
default_ttl: Some(Duration::from_secs(60)),
|
||||
}),
|
||||
Some("facade and native backend capacities must match")
|
||||
);
|
||||
assert_eq!(
|
||||
memory().mismatch(&BackendIdentity::Memory {
|
||||
capacity,
|
||||
max_entry_bytes: Some(1),
|
||||
default_ttl: Some(Duration::from_secs(61)),
|
||||
}),
|
||||
Some("facade and native backend default TTLs must match")
|
||||
);
|
||||
assert_eq!(
|
||||
redis().mismatch(&BackendIdentity::Redis {
|
||||
topology: RedisTopology::Cluster {
|
||||
startup_nodes: vec![RedisNode {
|
||||
host: "node".into(),
|
||||
port: 7000,
|
||||
}],
|
||||
},
|
||||
namespace: None,
|
||||
default_ttl: Some(Duration::from_secs(60)),
|
||||
}),
|
||||
Some("facade and native backend topologies must match")
|
||||
);
|
||||
assert_eq!(
|
||||
s3().mismatch(&BackendIdentity::S3 {
|
||||
bucket: "bucket".into(),
|
||||
key_prefix: "cache/".into(),
|
||||
region: "us-east-1".into(),
|
||||
endpoint: Some("http://localhost:9000".into()),
|
||||
}),
|
||||
Some("facade and native backend endpoints must match")
|
||||
);
|
||||
assert_eq!(
|
||||
gcs().mismatch(&BackendIdentity::Gcs {
|
||||
bucket_name: "bucket".into(),
|
||||
key_prefix: "cache/".into(),
|
||||
path_service_account: None,
|
||||
}),
|
||||
Some("facade and native backend credentials must match")
|
||||
);
|
||||
assert_eq!(
|
||||
azure().mismatch(&BackendIdentity::AzureBlob {
|
||||
account_url: "https://account.blob.core.windows.net".into(),
|
||||
container: "other".into(),
|
||||
}),
|
||||
Some("facade and native backend containers must match")
|
||||
);
|
||||
assert_eq!(
|
||||
valkey_semantic().mismatch(&BackendIdentity::ValkeySemantic {
|
||||
index_name: "idx".into(),
|
||||
similarity_threshold: 0.9,
|
||||
}),
|
||||
Some("facade and native semantic settings must match")
|
||||
);
|
||||
assert_eq!(
|
||||
qdrant().mismatch(&BackendIdentity::QdrantSemantic {
|
||||
collection_name: "collection".into(),
|
||||
similarity_threshold: 0.8,
|
||||
vector_size: 1536,
|
||||
embedding_model: "text-embedding-3-large".into(),
|
||||
}),
|
||||
Some("facade and native backend embedding models must match")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_directories_compare_canonically() {
|
||||
let directory = std::env::temp_dir();
|
||||
let mut indirect = directory.clone();
|
||||
indirect.push(".");
|
||||
assert_eq!(
|
||||
BackendIdentity::Disk {
|
||||
directory: directory.clone()
|
||||
}
|
||||
.mismatch(&BackendIdentity::Disk {
|
||||
directory: indirect
|
||||
}),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
BackendIdentity::Disk { directory }.mismatch(&BackendIdentity::Disk {
|
||||
directory: "/definitely/missing".into()
|
||||
}),
|
||||
Some("facade and native backend directories must match")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,11 +5,11 @@ mod embedder;
|
|||
mod facade;
|
||||
mod future;
|
||||
mod handle;
|
||||
mod identity;
|
||||
mod native;
|
||||
mod request;
|
||||
mod resolver;
|
||||
mod semantic;
|
||||
mod semantic_step;
|
||||
|
||||
use litellm_cache::Error;
|
||||
use pyo3::{
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,7 +1,7 @@
|
|||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use litellm_cache::ExactCacheContext;
|
||||
use litellm_cache_response::{CacheControls, CacheKeyInput, ResponseCacheRequest};
|
||||
use litellm_cache::{ExactCacheContext, SemanticCacheContext};
|
||||
use litellm_cache_response::{CacheControls, CacheKeyField, CacheKeyInput, ResponseCacheRequest};
|
||||
use litellm_host_python::from_py;
|
||||
use pyo3::{exceptions::PyValueError, prelude::*};
|
||||
use serde::Deserialize;
|
||||
|
|
@ -36,6 +36,99 @@ pub(super) struct NativeRequest {
|
|||
pub(super) scope: Option<String>,
|
||||
}
|
||||
|
||||
impl NativeRequest {
|
||||
pub(super) fn exact(&self) -> ResponseCacheRequest<ExactCacheContext> {
|
||||
ResponseCacheRequest {
|
||||
key: self.key.clone(),
|
||||
controls: self.controls,
|
||||
context: ExactCacheContext { ttl: self.ttl },
|
||||
max_age: self.max_age,
|
||||
}
|
||||
}
|
||||
|
||||
/// The request as a semantic backend that keys on the caller's scope sees it.
|
||||
pub(super) fn semantic(&self) -> ResponseCacheRequest<SemanticCacheContext> {
|
||||
self.semantic_with(self.key.clone(), self.scope.clone())
|
||||
}
|
||||
|
||||
/// The request keyed the way Python's Valkey semantic cache keys it: prompt fields drop out
|
||||
/// and the tenant identifiers for `scope` join the key.
|
||||
pub(super) fn scoped_semantic(
|
||||
&self,
|
||||
scope: &str,
|
||||
) -> ResponseCacheRequest<SemanticCacheContext> {
|
||||
self.semantic_with(semantic_key(self, scope), Some(scope.to_owned()))
|
||||
}
|
||||
|
||||
fn semantic_with(
|
||||
&self,
|
||||
key: CacheKeyInput,
|
||||
scope: Option<String>,
|
||||
) -> ResponseCacheRequest<SemanticCacheContext> {
|
||||
ResponseCacheRequest {
|
||||
key,
|
||||
controls: self.controls,
|
||||
context: SemanticCacheContext {
|
||||
input: self.input.clone(),
|
||||
messages: self.messages.clone(),
|
||||
metadata: self.metadata.clone(),
|
||||
scope,
|
||||
ttl: self.ttl,
|
||||
},
|
||||
max_age: self.max_age,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn semantic_key(request: &NativeRequest, scope: &str) -> CacheKeyInput {
|
||||
let mut key = request.key.clone();
|
||||
if key.preset.is_some() {
|
||||
return key;
|
||||
}
|
||||
key.fields
|
||||
.retain(|field| !matches!(field.name.as_str(), "messages" | "prompt" | "input"));
|
||||
const TENANT: [&str; 3] = [
|
||||
"user_api_key",
|
||||
"user_api_key_team_id",
|
||||
"user_api_key_org_id",
|
||||
];
|
||||
let end_user = (scope == "end_user").then_some("user_api_key_end_user_id");
|
||||
for name in TENANT.into_iter().chain(end_user) {
|
||||
let sources = [
|
||||
request.metadata.as_ref(),
|
||||
request.litellm_metadata.as_ref(),
|
||||
request
|
||||
.litellm_params
|
||||
.as_ref()
|
||||
.and_then(|params| params.get("metadata")),
|
||||
request
|
||||
.litellm_params
|
||||
.as_ref()
|
||||
.and_then(|params| params.get("litellm_metadata")),
|
||||
];
|
||||
let Some(value) = sources.into_iter().flatten().find_map(|source| {
|
||||
source
|
||||
.as_object()
|
||||
.and_then(|values| values.get(name))
|
||||
.filter(|value| !value.is_null())
|
||||
}) else {
|
||||
continue;
|
||||
};
|
||||
let value = match value {
|
||||
Value::Null => continue,
|
||||
Value::String(text) => text.clone(),
|
||||
other => other.to_string(),
|
||||
};
|
||||
key.fields.push(CacheKeyField {
|
||||
name: name.to_owned(),
|
||||
value: Some(value),
|
||||
api_parameter: true,
|
||||
internal_parameter: false,
|
||||
});
|
||||
}
|
||||
key
|
||||
}
|
||||
|
||||
pub(super) fn request(value: &Bound<'_, PyAny>) -> PyResult<NativeRequest> {
|
||||
let input: RequestInput = from_py(value)?;
|
||||
request_input(input)
|
||||
|
|
@ -76,3 +169,90 @@ pub(super) fn now() -> Duration {
|
|||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_cache_response::{CacheControls, CacheKeyInput, cache_key};
|
||||
use serde_json::json;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn native_request(key: CacheKeyInput, metadata: Value) -> NativeRequest {
|
||||
NativeRequest {
|
||||
key,
|
||||
controls: CacheControls::default(),
|
||||
ttl: None,
|
||||
max_age: None,
|
||||
messages: Some(json!([{"role": "user", "content": "prompt"}])),
|
||||
input: None,
|
||||
metadata: Some(metadata),
|
||||
litellm_metadata: None,
|
||||
litellm_params: None,
|
||||
scope: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_key_matches_python_scope_material() {
|
||||
let key = CacheKeyInput {
|
||||
fields: vec![
|
||||
CacheKeyField {
|
||||
name: "model".to_owned(),
|
||||
value: Some("gpt-4.1".to_owned()),
|
||||
api_parameter: true,
|
||||
internal_parameter: false,
|
||||
},
|
||||
CacheKeyField {
|
||||
name: "messages".to_owned(),
|
||||
value: Some("prompt".to_owned()),
|
||||
api_parameter: true,
|
||||
internal_parameter: false,
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
let request = native_request(
|
||||
key,
|
||||
json!({"user_api_key": "k1", "user_api_key_team_id": null}),
|
||||
);
|
||||
let expected = format!("{:x}", Sha256::digest(b"model: gpt-4.1user_api_key: k1"));
|
||||
assert_eq!(cache_key(&semantic_key(&request, "key")), expected);
|
||||
assert_eq!(cache_key(&request.scoped_semantic("key").key), expected);
|
||||
|
||||
let end_user_request = native_request(
|
||||
request.key.clone(),
|
||||
json!({"user_api_key": "k1", "user_api_key_end_user_id": "u1"}),
|
||||
);
|
||||
let expected = format!(
|
||||
"{:x}",
|
||||
Sha256::digest(b"model: gpt-4.1user_api_key: k1user_api_key_end_user_id: u1")
|
||||
);
|
||||
assert_eq!(
|
||||
cache_key(&semantic_key(&end_user_request, "end_user")),
|
||||
expected
|
||||
);
|
||||
|
||||
let preset_request = native_request(
|
||||
CacheKeyInput {
|
||||
preset: Some("preset-key".to_owned()),
|
||||
..Default::default()
|
||||
},
|
||||
json!({"user_api_key": "k1"}),
|
||||
);
|
||||
assert_eq!(
|
||||
semantic_key(&preset_request, "end_user").preset.as_deref(),
|
||||
Some("preset-key")
|
||||
);
|
||||
assert!(semantic_key(&preset_request, "end_user").fields.is_empty());
|
||||
assert_eq!(preset_request.semantic().context.scope, None);
|
||||
assert_eq!(
|
||||
preset_request
|
||||
.scoped_semantic("end_user")
|
||||
.context
|
||||
.scope
|
||||
.as_deref(),
|
||||
Some("end_user")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
use std::collections::VecDeque;
|
||||
use std::{collections::VecDeque, time::Duration};
|
||||
|
||||
use litellm_cache::Error;
|
||||
use litellm_cache_redis_semantic::prompt_from_context;
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
|
|
@ -23,29 +22,104 @@ pub(super) enum SemanticOperation {
|
|||
StoreBatch(VecDeque<(NativeRequest, Value)>),
|
||||
}
|
||||
|
||||
/// What an exception from the Python embedder means for the operation.
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) enum EmbeddingFailure {
|
||||
/// Raise the Python exception unchanged.
|
||||
Propagate,
|
||||
/// Treat the embedding as unavailable and let the backend report that.
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
enum Phase {
|
||||
Start,
|
||||
AwaitingEmbedding,
|
||||
AwaitingBackend,
|
||||
}
|
||||
|
||||
pub(super) struct SemanticBody {
|
||||
/// Runs a semantic cache operation whose embedding comes from Python: await the Python
|
||||
/// embedder in the caller's event loop, seed the native backend with the vector, await the
|
||||
/// backend, and repeat for each entry of a batch.
|
||||
pub(super) struct SemanticExecution {
|
||||
service: NativeResponseCache,
|
||||
embedder: PythonEmbedder,
|
||||
failure: EmbeddingFailure,
|
||||
operation: SemanticOperation,
|
||||
pending: Option<(NativeRequest, Option<Value>)>,
|
||||
phase: Phase,
|
||||
now: Duration,
|
||||
}
|
||||
|
||||
impl SemanticBody {
|
||||
pub(super) fn new(service: NativeResponseCache, operation: SemanticOperation) -> Self {
|
||||
impl SemanticExecution {
|
||||
pub(super) fn new(
|
||||
service: NativeResponseCache,
|
||||
embedder: PythonEmbedder,
|
||||
failure: EmbeddingFailure,
|
||||
operation: SemanticOperation,
|
||||
) -> Self {
|
||||
Self {
|
||||
service,
|
||||
embedder,
|
||||
failure,
|
||||
operation,
|
||||
pending: None,
|
||||
phase: Phase::Start,
|
||||
now: now(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Takes the next entry of the operation; `None` once a batch is exhausted.
|
||||
fn next_pending(&mut self) -> Option<(NativeRequest, Option<Value>)> {
|
||||
match &mut self.operation {
|
||||
SemanticOperation::Lookup(request) => Some((request.clone(), None)),
|
||||
SemanticOperation::Store(request, response) => {
|
||||
Some((request.clone(), Some(std::mem::take(response))))
|
||||
}
|
||||
SemanticOperation::StoreBatch(queue) => queue
|
||||
.pop_front()
|
||||
.map(|(request, response)| (request, Some(response))),
|
||||
}
|
||||
}
|
||||
|
||||
fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
|
||||
let Some(pending) = self.next_pending() else {
|
||||
return Ok(ExecutionStep::Return(py.None()));
|
||||
};
|
||||
let (request, response) = &pending;
|
||||
let enabled = match response {
|
||||
None => request.controls.reads(),
|
||||
Some(_) => request.controls.writes(),
|
||||
};
|
||||
let input = enabled
|
||||
.then(|| self.service.embedding_input(request))
|
||||
.flatten();
|
||||
self.pending = Some(pending);
|
||||
let Some(input) = input else {
|
||||
return self.backend_step(py, Err(Error::Unavailable));
|
||||
};
|
||||
let awaitable =
|
||||
self.embedder
|
||||
.async_embedding(py, &input.prompt, input.metadata.as_ref())?;
|
||||
self.phase = Phase::AwaitingEmbedding;
|
||||
Ok(ExecutionStep::Await(awaitable))
|
||||
}
|
||||
|
||||
fn embedded(&mut self, py: Python<'_>, result: PyResult<Py<PyAny>>) -> PyResult<ExecutionStep> {
|
||||
let seed = match result {
|
||||
Ok(vector) => {
|
||||
PythonEmbedder::extract(vector.into_bound(py)).map_err(|_| Error::Unavailable)
|
||||
}
|
||||
Err(error) => match self.failure {
|
||||
EmbeddingFailure::Propagate => return Err(error),
|
||||
EmbeddingFailure::Unavailable if error.is_instance_of::<PyException>(py) => {
|
||||
Err(Error::Unavailable)
|
||||
}
|
||||
EmbeddingFailure::Unavailable => return Err(error),
|
||||
},
|
||||
};
|
||||
self.backend_step(py, seed)
|
||||
}
|
||||
|
||||
fn backend_step(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
|
|
@ -56,11 +130,12 @@ impl SemanticBody {
|
|||
PyRuntimeError::new_err("semantic execution resumed without a pending operation")
|
||||
})?;
|
||||
let service = self.service.clone();
|
||||
let now = self.now;
|
||||
let future = async move {
|
||||
match response {
|
||||
None => service.async_lookup(&request, now()).await,
|
||||
None => service.async_lookup(&request, now).await,
|
||||
Some(response) => service
|
||||
.async_store(&request, response, now())
|
||||
.async_store(&request, response, now)
|
||||
.await
|
||||
.map(|_| None),
|
||||
}
|
||||
|
|
@ -68,106 +143,45 @@ impl SemanticBody {
|
|||
let awaitable = run_async(py, with_prepared_embedding(seed, future), cache_error)?;
|
||||
Ok(ExecutionStep::Await(awaitable.unbind()))
|
||||
}
|
||||
|
||||
fn resume_py(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
result: Option<PyResult<Py<PyAny>>>,
|
||||
) -> PyResult<ExecutionStep> {
|
||||
match (&self.phase, result) {
|
||||
(Phase::Start, None) => self.start(py),
|
||||
(Phase::AwaitingEmbedding, Some(result)) => self.embedded(py, result),
|
||||
(Phase::AwaitingBackend, Some(Err(error))) => Err(error),
|
||||
(Phase::AwaitingBackend, Some(Ok(value))) => {
|
||||
let more = matches!(
|
||||
&self.operation,
|
||||
SemanticOperation::StoreBatch(queue) if !queue.is_empty()
|
||||
);
|
||||
if more {
|
||||
self.phase = Phase::Start;
|
||||
return self.start(py);
|
||||
}
|
||||
Ok(ExecutionStep::Return(value))
|
||||
}
|
||||
_ => Err(PyRuntimeError::new_err(
|
||||
"invalid semantic cache execution state",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionBody for SemanticBody {
|
||||
fn resume(&mut self, mut result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
|
||||
Python::attach(|py| {
|
||||
loop {
|
||||
match self.phase {
|
||||
Phase::Start => {
|
||||
if result.is_some() {
|
||||
return Err(PyRuntimeError::new_err(
|
||||
"semantic execution received a result before starting",
|
||||
));
|
||||
}
|
||||
if self.pending.is_none() {
|
||||
match &mut self.operation {
|
||||
SemanticOperation::Lookup(request) => {
|
||||
self.pending = Some((request.clone(), None));
|
||||
}
|
||||
SemanticOperation::Store(request, response) => {
|
||||
let response = std::mem::replace(response, Value::Null);
|
||||
self.pending = Some((request.clone(), Some(response)));
|
||||
}
|
||||
SemanticOperation::StoreBatch(queue) => {
|
||||
let Some((request, response)) = queue.pop_front() else {
|
||||
return Ok(ExecutionStep::Return(py.None()));
|
||||
};
|
||||
self.pending = Some((request, Some(response)));
|
||||
}
|
||||
}
|
||||
}
|
||||
let (request, _) = self.pending.as_ref().ok_or_else(|| {
|
||||
PyRuntimeError::new_err("semantic execution has no pending operation")
|
||||
})?;
|
||||
let semantic = NativeResponseCache::semantic_request(request);
|
||||
let Some(prompt) = prompt_from_context(&semantic.context) else {
|
||||
return self.backend_step(py, Err(Error::Unavailable));
|
||||
};
|
||||
let embedder = self.service.semantic_embedder().ok_or_else(|| {
|
||||
PyRuntimeError::new_err(
|
||||
"semantic execution requires a redis-semantic backend",
|
||||
)
|
||||
})?;
|
||||
let coroutine = embedder.async_embedding_coroutine(
|
||||
py,
|
||||
&prompt,
|
||||
semantic.context.metadata.as_ref(),
|
||||
)?;
|
||||
self.phase = Phase::AwaitingEmbedding;
|
||||
return Ok(ExecutionStep::Await(coroutine));
|
||||
}
|
||||
Phase::AwaitingEmbedding => {
|
||||
let result = result.take().ok_or_else(|| {
|
||||
PyRuntimeError::new_err(
|
||||
"semantic execution expected an embedding result",
|
||||
)
|
||||
})?;
|
||||
let seed = match result {
|
||||
Ok(value) => PythonEmbedder::extract(value.into_bound(py))
|
||||
.map_err(|_| Error::Unavailable),
|
||||
Err(error) => {
|
||||
if !error.is_instance_of::<PyException>(py) {
|
||||
return Err(error);
|
||||
}
|
||||
Err(Error::Unavailable)
|
||||
}
|
||||
};
|
||||
return self.backend_step(py, seed);
|
||||
}
|
||||
Phase::AwaitingBackend => {
|
||||
let result = result.take().ok_or_else(|| {
|
||||
PyRuntimeError::new_err("semantic execution expected a backend result")
|
||||
})?;
|
||||
let value = match result {
|
||||
Ok(value) => value,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let more = matches!(
|
||||
&self.operation,
|
||||
SemanticOperation::StoreBatch(queue) if !queue.is_empty()
|
||||
);
|
||||
if more {
|
||||
self.phase = Phase::Start;
|
||||
continue;
|
||||
}
|
||||
return Ok(ExecutionStep::Return(value));
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
impl ExecutionBody for SemanticExecution {
|
||||
fn resume(&mut self, result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
|
||||
Python::attach(|py| self.resume_py(py, result))
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
if let Some(embedder) = self.service.semantic_embedder() {
|
||||
embedder.traverse(visit)?;
|
||||
}
|
||||
Ok(())
|
||||
self.embedder.traverse(visit)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn drive(py: Python<'_>, body: SemanticBody) -> PyResult<Bound<'_, PyAny>> {
|
||||
pub(super) fn drive(py: Python<'_>, body: SemanticExecution) -> PyResult<Bound<'_, PyAny>> {
|
||||
let execution = Py::new(py, Execution::new(body))?;
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("drive")?
|
||||
|
|
|
|||
|
|
@ -1,249 +0,0 @@
|
|||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use litellm_cache::SemanticCacheContext;
|
||||
use litellm_cache_response::{ResponseCache, ResponseCacheCodec, ResponseCacheRequest};
|
||||
use litellm_cache_valkey_semantic::{PreparedEmbedding, ValkeySemanticCache, prompt_from_context};
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
|
||||
use pyo3::{PyTraverseError, PyVisit, exceptions::PyRuntimeError, prelude::*};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::{cache_error, embedder::PythonEmbedder};
|
||||
|
||||
pub(super) enum Op {
|
||||
Lookup,
|
||||
Store(Value),
|
||||
StoreBatch(Vec<Value>),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum State {
|
||||
Start,
|
||||
AwaitingEmbedding,
|
||||
AwaitingStorage,
|
||||
Done,
|
||||
}
|
||||
|
||||
pub(super) struct SemanticEmbedExecution {
|
||||
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
|
||||
embedder: PythonEmbedder,
|
||||
requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
|
||||
op: Op,
|
||||
now: Option<Duration>,
|
||||
prepared: Vec<Option<Vec<f32>>>,
|
||||
index: usize,
|
||||
state: State,
|
||||
}
|
||||
|
||||
impl SemanticEmbedExecution {
|
||||
pub(super) fn lookup(
|
||||
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
|
||||
embedder: PythonEmbedder,
|
||||
request: ResponseCacheRequest<SemanticCacheContext>,
|
||||
) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
embedder,
|
||||
requests: vec![request],
|
||||
op: Op::Lookup,
|
||||
now: None,
|
||||
prepared: vec![None],
|
||||
index: 0,
|
||||
state: State::Start,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn store(
|
||||
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
|
||||
embedder: PythonEmbedder,
|
||||
request: ResponseCacheRequest<SemanticCacheContext>,
|
||||
response: Value,
|
||||
) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
embedder,
|
||||
requests: vec![request],
|
||||
op: Op::Store(response),
|
||||
now: None,
|
||||
prepared: vec![None],
|
||||
index: 0,
|
||||
state: State::Start,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn store_batch(
|
||||
backend: Arc<ValkeySemanticCache<PythonEmbedder, ResponseCacheCodec>>,
|
||||
embedder: PythonEmbedder,
|
||||
requests: Vec<ResponseCacheRequest<SemanticCacheContext>>,
|
||||
responses: Vec<Value>,
|
||||
) -> Self {
|
||||
Self {
|
||||
backend,
|
||||
embedder,
|
||||
prepared: vec![None; requests.len()],
|
||||
requests,
|
||||
op: Op::StoreBatch(responses),
|
||||
now: None,
|
||||
index: 0,
|
||||
state: State::Start,
|
||||
}
|
||||
}
|
||||
|
||||
fn start(&mut self, py: Python<'_>) -> PyResult<ExecutionStep> {
|
||||
if self.now.is_none() {
|
||||
self.now = Some(super::request::now());
|
||||
}
|
||||
while self.index < self.requests.len() {
|
||||
let request = &self.requests[self.index];
|
||||
let enabled = match &self.op {
|
||||
Op::Lookup => request.controls.reads(),
|
||||
Op::Store(_) | Op::StoreBatch(_) => request.controls.writes(),
|
||||
};
|
||||
if !enabled {
|
||||
self.index += 1;
|
||||
continue;
|
||||
}
|
||||
let Some(prompt) = prompt_from_context(&request.context) else {
|
||||
self.index += 1;
|
||||
continue;
|
||||
};
|
||||
let metadata = request.context.metadata.clone();
|
||||
let awaitable = self
|
||||
.embedder
|
||||
.async_embed_awaitable(py, &prompt, &metadata)?;
|
||||
self.state = State::AwaitingEmbedding;
|
||||
return Ok(ExecutionStep::Await(awaitable.unbind()));
|
||||
}
|
||||
self.state = State::AwaitingStorage;
|
||||
self.storage_step(py)
|
||||
}
|
||||
|
||||
fn storage_step(&self, py: Python<'_>) -> PyResult<ExecutionStep> {
|
||||
let requests = self.requests.clone();
|
||||
let prepared = self.prepared.clone();
|
||||
let backend = Arc::clone(&self.backend);
|
||||
let now = self
|
||||
.now
|
||||
.ok_or_else(|| PyRuntimeError::new_err("semantic cache timestamp is unavailable"))?;
|
||||
let awaitable = match &self.op {
|
||||
Op::Lookup => {
|
||||
let Some(request) = requests.into_iter().next() else {
|
||||
return Err(PyRuntimeError::new_err(
|
||||
"semantic lookup requires one request",
|
||||
));
|
||||
};
|
||||
match prepared.into_iter().next().flatten() {
|
||||
Some(values) => {
|
||||
let backend = backend.with_embedder(PreparedEmbedding(values));
|
||||
let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
|
||||
run_async(
|
||||
py,
|
||||
async move { cache.async_lookup(&request, now).await },
|
||||
cache_error,
|
||||
)?
|
||||
}
|
||||
None => {
|
||||
let cache = Arc::new(ResponseCache::new(backend));
|
||||
run_async(
|
||||
py,
|
||||
async move { cache.async_lookup(&request, now).await },
|
||||
cache_error,
|
||||
)?
|
||||
}
|
||||
}
|
||||
}
|
||||
Op::Store(response) => {
|
||||
let Some(request) = requests.into_iter().next() else {
|
||||
return Err(PyRuntimeError::new_err(
|
||||
"semantic store requires one request",
|
||||
));
|
||||
};
|
||||
let response = response.clone();
|
||||
match prepared.into_iter().next().flatten() {
|
||||
Some(values) => {
|
||||
let backend = backend.with_embedder(PreparedEmbedding(values));
|
||||
let cache = Arc::new(ResponseCache::new(Arc::new(backend)));
|
||||
run_async(
|
||||
py,
|
||||
async move { cache.async_store(&request, response, now).await },
|
||||
cache_error,
|
||||
)?
|
||||
}
|
||||
None => {
|
||||
let cache = Arc::new(ResponseCache::new(backend));
|
||||
run_async(
|
||||
py,
|
||||
async move { cache.async_store(&request, response, now).await },
|
||||
cache_error,
|
||||
)?
|
||||
}
|
||||
}
|
||||
}
|
||||
Op::StoreBatch(responses) => {
|
||||
let responses = responses.clone();
|
||||
run_async(
|
||||
py,
|
||||
async move {
|
||||
for ((request, response), prepared) in
|
||||
requests.into_iter().zip(responses).zip(prepared)
|
||||
{
|
||||
let Some(values) = prepared else {
|
||||
continue;
|
||||
};
|
||||
let backend = backend.with_embedder(PreparedEmbedding(values));
|
||||
let cache = ResponseCache::new(Arc::new(backend));
|
||||
cache.async_store(&request, response, now).await?;
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
cache_error,
|
||||
)?
|
||||
}
|
||||
};
|
||||
Ok(ExecutionStep::Await(awaitable.unbind()))
|
||||
}
|
||||
|
||||
fn resume_py(
|
||||
&mut self,
|
||||
py: Python<'_>,
|
||||
result: Option<PyResult<Py<PyAny>>>,
|
||||
) -> PyResult<ExecutionStep> {
|
||||
match (self.state, result) {
|
||||
(State::Start, None) => self.start(py),
|
||||
(State::AwaitingEmbedding, Some(Ok(value))) => {
|
||||
let values = value.bind(py).extract::<Vec<f64>>()?;
|
||||
self.prepared[self.index] =
|
||||
Some(values.into_iter().map(|value| value as f32).collect());
|
||||
self.index += 1;
|
||||
self.start(py)
|
||||
}
|
||||
(State::AwaitingStorage, Some(Ok(value))) => {
|
||||
self.state = State::Done;
|
||||
Ok(ExecutionStep::Return(value))
|
||||
}
|
||||
(_, Some(Err(error))) => Err(error),
|
||||
_ => Err(PyRuntimeError::new_err(
|
||||
"invalid semantic cache execution state",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ExecutionBody for SemanticEmbedExecution {
|
||||
fn resume(&mut self, result: Option<PyResult<Py<PyAny>>>) -> PyResult<ExecutionStep> {
|
||||
Python::attach(|py| self.resume_py(py, result))
|
||||
}
|
||||
|
||||
fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
|
||||
self.embedder.traverse(visit)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn drive_semantic<'py>(
|
||||
py: Python<'py>,
|
||||
body: SemanticEmbedExecution,
|
||||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let execution = Py::new(py, Execution::new(body))?;
|
||||
py.import("litellm.rust_bridge.lifecycle")?
|
||||
.getattr("drive")?
|
||||
.call1((execution,))
|
||||
}
|
||||
|
|
@ -1,7 +1,4 @@
|
|||
use std::collections::BTreeSet;
|
||||
|
||||
use litellm_core_utils::serde_compat::parse_str_bool;
|
||||
use litellm_http::SslVerify;
|
||||
use pyo3::{
|
||||
exceptions::{PyAttributeError, PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
|
|
@ -33,36 +30,52 @@ impl From<ProjectionError> for PyErr {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Truthy(pub bool);
|
||||
pub(crate) struct ExactTrue(pub bool);
|
||||
pub(crate) struct StrBool(pub Option<bool>);
|
||||
pub(crate) struct OptionalStrictString(pub Option<String>);
|
||||
pub(crate) struct FalsyOptionalString(pub Option<String>);
|
||||
pub(crate) struct TuningString(pub Option<String>);
|
||||
pub(crate) struct StringCollection(pub Vec<String>);
|
||||
pub(crate) struct SslVerifyInput(pub Option<SslVerify>);
|
||||
pub(crate) struct FieldSpec<T> {
|
||||
name: &'static str,
|
||||
decode: fn(&Field<'_>) -> Result<T, ProjectionError>,
|
||||
}
|
||||
|
||||
impl<T> FieldSpec<T> {
|
||||
pub(crate) const fn new(
|
||||
name: &'static str,
|
||||
decode: fn(&Field<'_>) -> Result<T, ProjectionError>,
|
||||
) -> Self {
|
||||
Self { name, decode }
|
||||
}
|
||||
|
||||
pub(crate) fn read(
|
||||
&self,
|
||||
snapshot: &Bound<'_, PyAny>,
|
||||
group: &'static str,
|
||||
) -> Result<T, ProjectionError> {
|
||||
(self.decode)(&Field::read(snapshot, group, self.name)?)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct Field<'py> {
|
||||
path: &'static str,
|
||||
group: &'static str,
|
||||
name: &'static str,
|
||||
value: Bound<'py, PyAny>,
|
||||
}
|
||||
|
||||
impl<'py> Field<'py> {
|
||||
pub(crate) fn new(path: &'static str, value: Bound<'py, PyAny>) -> Self {
|
||||
Self { path, value }
|
||||
pub(crate) fn new(group: &'static str, name: &'static str, value: Bound<'py, PyAny>) -> Self {
|
||||
Self { group, name, value }
|
||||
}
|
||||
|
||||
/// Reads `snapshot.<name>`, distinguishing a field the accessor never declared from a
|
||||
/// descriptor that raised `AttributeError`.
|
||||
pub(crate) fn read(
|
||||
snapshot: &Bound<'py, PyAny>,
|
||||
path: &'static str,
|
||||
group: &'static str,
|
||||
name: &'static str,
|
||||
) -> Result<Self, ProjectionError> {
|
||||
let name = path.rsplit('.').next().unwrap_or(path);
|
||||
match snapshot.getattr(name) {
|
||||
Ok(value) => Ok(Self::new(path, value)),
|
||||
Ok(value) => Ok(Self::new(group, name, value)),
|
||||
Err(error) if error.is_instance_of::<PyAttributeError>(snapshot.py()) => {
|
||||
match Self::missing_field(snapshot, name) {
|
||||
Ok(true) => Err(ProjectionError::InternalSchemaFailure(format!(
|
||||
"{path}: missing snapshot field"
|
||||
"{group}.{name}: missing snapshot field"
|
||||
))),
|
||||
_ => Err(error.into()),
|
||||
}
|
||||
|
|
@ -84,27 +97,49 @@ impl<'py> Field<'py> {
|
|||
&& getter.is(object.getattr("__getattribute__")?))
|
||||
}
|
||||
|
||||
fn expected(&self, expected: &'static str) -> Result<String, ProjectionError> {
|
||||
pub(crate) fn path(&self) -> String {
|
||||
format!("{}.{}", self.group, self.name)
|
||||
}
|
||||
|
||||
/// A member of this field's collection, reported under the same path.
|
||||
pub(crate) fn member(&self, value: Bound<'py, PyAny>) -> Self {
|
||||
Self::new(self.group, self.name, value)
|
||||
}
|
||||
|
||||
pub(crate) fn expected(&self, expected: &str) -> Result<String, ProjectionError> {
|
||||
Ok(format!(
|
||||
"{}: expected {expected}, got {}",
|
||||
self.path,
|
||||
self.path(),
|
||||
self.value.get_type().name()?
|
||||
))
|
||||
}
|
||||
|
||||
fn invalid(&self, expected: &'static str) -> ProjectionError {
|
||||
pub(crate) fn invalid(&self, expected: &str) -> ProjectionError {
|
||||
match self.expected(expected) {
|
||||
Ok(message) => ProjectionError::InvalidConfiguration(message),
|
||||
Err(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn truthy(&self) -> Result<Truthy, ProjectionError> {
|
||||
Ok(Truthy(self.value.is_truthy()?))
|
||||
pub(crate) fn value(&self) -> &Bound<'py, PyAny> {
|
||||
&self.value
|
||||
}
|
||||
|
||||
pub(crate) fn exact_true(&self) -> ExactTrue {
|
||||
ExactTrue(self.value.is(PyBool::new(self.value.py(), true)))
|
||||
pub(crate) fn truthy(&self) -> Result<bool, ProjectionError> {
|
||||
Ok(self.value.is_truthy()?)
|
||||
}
|
||||
|
||||
pub(crate) fn exact_true(&self) -> bool {
|
||||
self.value.is(PyBool::new(self.value.py(), true))
|
||||
}
|
||||
|
||||
pub(crate) fn schema_bool(&self) -> Result<bool, ProjectionError> {
|
||||
if !self.value.is_instance_of::<PyBool>() {
|
||||
return Err(ProjectionError::InternalSchemaFailure(
|
||||
self.expected("a Boolean")?,
|
||||
));
|
||||
}
|
||||
Ok(self.exact_true())
|
||||
}
|
||||
|
||||
pub(crate) fn strict_string(&self) -> Result<String, ProjectionError> {
|
||||
|
|
@ -124,108 +159,366 @@ impl<'py> Field<'py> {
|
|||
self.strict_string()
|
||||
}
|
||||
|
||||
pub(crate) fn schema_bool(&self) -> Result<bool, ProjectionError> {
|
||||
if !self.value.is_instance_of::<PyBool>() {
|
||||
return Err(ProjectionError::InternalSchemaFailure(
|
||||
self.expected("a Boolean")?,
|
||||
));
|
||||
}
|
||||
Ok(self.exact_true().0)
|
||||
}
|
||||
|
||||
pub(crate) fn str_bool(&self) -> Result<StrBool, ProjectionError> {
|
||||
pub(crate) fn str_bool(&self) -> Result<Option<bool>, ProjectionError> {
|
||||
if self.value.is_none() {
|
||||
return Ok(StrBool(None));
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(StrBool(parse_str_bool(&self.strict_string()?)))
|
||||
Ok(parse_str_bool(&self.strict_string()?))
|
||||
}
|
||||
|
||||
pub(crate) fn optional_strict_string(&self) -> Result<OptionalStrictString, ProjectionError> {
|
||||
pub(crate) fn optional_strict_string(&self) -> Result<Option<String>, ProjectionError> {
|
||||
if self.value.is_none() {
|
||||
return Ok(OptionalStrictString(None));
|
||||
return Ok(None);
|
||||
}
|
||||
self.strict_string().map(Some).map(OptionalStrictString)
|
||||
self.strict_string().map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn falsy_optional_string(&self) -> Result<FalsyOptionalString, ProjectionError> {
|
||||
if !self.truthy()?.0 {
|
||||
return Ok(FalsyOptionalString(None));
|
||||
pub(crate) fn falsy_optional_string(&self) -> Result<Option<String>, ProjectionError> {
|
||||
if !self.truthy()? {
|
||||
return Ok(None);
|
||||
}
|
||||
self.strict_string().map(Some).map(FalsyOptionalString)
|
||||
self.strict_string().map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn tuning_string(&self) -> Result<TuningString, ProjectionError> {
|
||||
if !self.truthy()?.0 || !self.value.is_instance_of::<PyString>() {
|
||||
return Ok(TuningString(None));
|
||||
pub(crate) fn tuning_string(&self) -> Result<Option<String>, ProjectionError> {
|
||||
if !self.truthy()? || !self.value.is_instance_of::<PyString>() {
|
||||
return Ok(None);
|
||||
}
|
||||
self.strict_string().map(Some).map(TuningString)
|
||||
self.strict_string().map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn string_collection(&self) -> Result<StringCollection, ProjectionError> {
|
||||
if !self.truthy()?.0 {
|
||||
return Ok(StringCollection(Vec::new()));
|
||||
pub(crate) fn string_collection(&self) -> Result<Vec<String>, ProjectionError> {
|
||||
if !self.truthy()? {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
if self.value.is_instance_of::<PyString>() {
|
||||
return self
|
||||
.strict_string()
|
||||
.map(|value| StringCollection(vec![value]));
|
||||
return self.strict_string().map(|value| vec![value]);
|
||||
}
|
||||
let values = self
|
||||
.value
|
||||
self.value
|
||||
.try_iter()?
|
||||
.filter_map(|item| {
|
||||
let member = match item {
|
||||
Ok(value) => Self::new(self.path, value),
|
||||
Ok(value) => self.member(value),
|
||||
Err(error) => return Some(Err(error.into())),
|
||||
};
|
||||
match member.truthy() {
|
||||
Ok(Truthy(false)) => None,
|
||||
Ok(Truthy(true)) => Some(member.strict_string()),
|
||||
Ok(false) => None,
|
||||
Ok(true) => Some(member.strict_string()),
|
||||
Err(error) => Some(Err(error)),
|
||||
}
|
||||
})
|
||||
.collect::<Result<Vec<_>, ProjectionError>>()?;
|
||||
Ok(StringCollection(values))
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn host_collection(&self) -> Result<StringCollection, ProjectionError> {
|
||||
let values = self
|
||||
.string_collection()?
|
||||
.0
|
||||
.into_iter()
|
||||
.map(|host| litellm_http::media::normalize_host(&host))
|
||||
.collect::<BTreeSet<_>>();
|
||||
Ok(StringCollection(values.into_iter().collect()))
|
||||
}
|
||||
|
||||
pub(crate) fn ssl_verify(&self) -> Result<SslVerifyInput, ProjectionError> {
|
||||
pub(crate) fn optional_string_collection(
|
||||
&self,
|
||||
) -> Result<Option<Vec<String>>, ProjectionError> {
|
||||
if self.value.is_none() {
|
||||
return Ok(SslVerifyInput(None));
|
||||
return Ok(None);
|
||||
}
|
||||
if self.value.is_instance_of::<PyBool>() {
|
||||
return Ok(SslVerifyInput(Some(if self.exact_true().0 {
|
||||
SslVerify::Enabled
|
||||
} else {
|
||||
SslVerify::Disabled
|
||||
})));
|
||||
}
|
||||
if self.value.is_instance_of::<PyString>() {
|
||||
let parsed = match self.str_bool()?.0 {
|
||||
Some(true) => SslVerify::Enabled,
|
||||
Some(false) => SslVerify::Disabled,
|
||||
None => SslVerify::CaBundle(self.strict_string()?.into()),
|
||||
};
|
||||
return Ok(SslVerifyInput(Some(parsed)));
|
||||
}
|
||||
let context = self.value.py().import("ssl")?.getattr("SSLContext")?;
|
||||
if self.value.is_instance(&context)? {
|
||||
return Err(ProjectionError::UnsupportedLiveObject(self.expected(
|
||||
"a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported",
|
||||
)?));
|
||||
}
|
||||
Err(self.invalid("a Boolean, Boolean string, CA path, or None"))
|
||||
self.string_collection().map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn python_binding(&self) -> Option<Py<PyAny>> {
|
||||
(!self.value.is_none()).then(|| self.value.clone().unbind())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
mod tests {
|
||||
use std::ffi::CString;
|
||||
|
||||
use pyo3::{
|
||||
exceptions::{PyLookupError, PyRuntimeError, PyValueError},
|
||||
types::PyDict,
|
||||
};
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
|
||||
py.eval(&CString::new(source).unwrap(), None, None).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("None", false, false)]
|
||||
#[case("False", false, false)]
|
||||
#[case("True", true, true)]
|
||||
#[case("0", false, false)]
|
||||
#[case("1", true, false)]
|
||||
#[case("''", false, false)]
|
||||
#[case("'false'", true, false)]
|
||||
#[case("[]", false, false)]
|
||||
#[case("[0]", true, false)]
|
||||
#[case("{}", false, false)]
|
||||
#[case("object()", true, false)]
|
||||
fn boolean_operations_have_distinct_python_semantics(
|
||||
#[case] source: &str,
|
||||
#[case] truth: bool,
|
||||
#[case] exact: bool,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let value = evaluate(py, source);
|
||||
let field = Field::new("test", "flag", value.clone());
|
||||
assert_eq!(field.truthy().unwrap(), truth);
|
||||
assert_eq!(field.exact_true(), exact);
|
||||
assert_eq!(
|
||||
field.truthy().unwrap(),
|
||||
py.import("builtins")
|
||||
.unwrap()
|
||||
.getattr("bool")
|
||||
.unwrap()
|
||||
.call1((value,))
|
||||
.unwrap()
|
||||
.extract::<bool>()
|
||||
.unwrap()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("None", Ok(None), Ok(None), Ok(None))]
|
||||
#[case("''", Ok(Some("")), Ok(None), Ok(None))]
|
||||
#[case(
|
||||
"' value '",
|
||||
Ok(Some(" value ")),
|
||||
Ok(Some(" value ")),
|
||||
Ok(Some(" value "))
|
||||
)]
|
||||
#[case("[]", Err(()), Ok(None), Ok(None))]
|
||||
#[case("0", Err(()), Ok(None), Ok(None))]
|
||||
#[case("1", Err(()), Err(()), Ok(None))]
|
||||
#[case("object()", Err(()), Err(()), Ok(None))]
|
||||
fn string_operations_do_not_conflate_absence_and_type_checks(
|
||||
#[case] source: &str,
|
||||
#[case] strict: Result<Option<&str>, ()>,
|
||||
#[case] fallback: Result<Option<&str>, ()>,
|
||||
#[case] tuning: Result<Option<&str>, ()>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let field = Field::new("test", "string", evaluate(py, source));
|
||||
let owned =
|
||||
|expected: Result<Option<&str>, ()>| expected.map(|value| value.map(str::to_owned));
|
||||
assert_eq!(
|
||||
field.optional_strict_string().map_err(|_| ()),
|
||||
owned(strict)
|
||||
);
|
||||
assert_eq!(
|
||||
field.falsy_optional_string().map_err(|_| ()),
|
||||
owned(fallback)
|
||||
);
|
||||
assert_eq!(field.tuning_string().map_err(|_| ()), owned(tuning));
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("None", None)]
|
||||
#[case("' True '", Some(true))]
|
||||
#[case("' fAlSe '", Some(false))]
|
||||
#[case("'yes'", None)]
|
||||
#[case("'1'", None)]
|
||||
#[case("'unknown'", None)]
|
||||
fn string_boolean_tokens_remain_separate_from_truthiness(
|
||||
#[case] source: &str,
|
||||
#[case] expected: Option<bool>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
assert_eq!(
|
||||
Field::new("test", "flag", evaluate(py, source))
|
||||
.str_bool()
|
||||
.unwrap(),
|
||||
expected
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
failure = LookupError('protocol failed')
|
||||
cause = ValueError('cause')
|
||||
context = RuntimeError('context')
|
||||
def fail():
|
||||
try:
|
||||
raise context
|
||||
except RuntimeError:
|
||||
raise failure from cause
|
||||
class Bool:
|
||||
def __bool__(self): return fail()
|
||||
class Length:
|
||||
def __len__(self): return fail()
|
||||
class Iter:
|
||||
def __iter__(self): return fail()
|
||||
class Next:
|
||||
def __iter__(self): return self
|
||||
def __next__(self): return fail()
|
||||
class Descriptor:
|
||||
@property
|
||||
def flag(self): return fail()
|
||||
values = (Bool(), Length(), Iter(), Next(), [Bool()])
|
||||
descriptor = Descriptor()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let values = locals.get_item("values").unwrap().unwrap();
|
||||
for value in values.try_iter().unwrap() {
|
||||
let error = Field::new("test", "flag", value.unwrap())
|
||||
.string_collection()
|
||||
.err()
|
||||
.unwrap();
|
||||
let error = PyErr::from(error);
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
assert!(error.is_instance_of::<PyLookupError>(py));
|
||||
assert!(error.traceback(py).is_some());
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.getattr("__cause__")
|
||||
.unwrap()
|
||||
.is(locals.get_item("cause").unwrap().unwrap())
|
||||
);
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.getattr("__context__")
|
||||
.unwrap()
|
||||
.is(locals.get_item("context").unwrap().unwrap())
|
||||
);
|
||||
}
|
||||
let error = Field::read(
|
||||
&locals.get_item("descriptor").unwrap().unwrap(),
|
||||
"test",
|
||||
"flag",
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(
|
||||
PyErr::from(error)
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_and_string_contents_do_not_invoke_unrelated_protocols() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
class Hostile:
|
||||
def __bool__(self): raise AssertionError('bool called')
|
||||
def __eq__(self, other): raise AssertionError('eq called')
|
||||
def __str__(self): raise AssertionError('str called')
|
||||
class Text(str):
|
||||
def __str__(self): raise AssertionError('str called')
|
||||
def strip(self): raise AssertionError('strip called')
|
||||
def lower(self): raise AssertionError('lower called')
|
||||
hostile = Hostile()
|
||||
text = Text(' False ')
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let hostile = Field::new("test", "flag", locals.get_item("hostile").unwrap().unwrap());
|
||||
assert!(!hostile.exact_true());
|
||||
assert!(matches!(
|
||||
hostile.strict_string(),
|
||||
Err(ProjectionError::InvalidConfiguration(_))
|
||||
));
|
||||
let text = Field::new("test", "flag", locals.get_item("text").unwrap().unwrap());
|
||||
assert_eq!(text.strict_string().unwrap(), " False ");
|
||||
assert_eq!(text.str_bool().unwrap(), Some(false));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
failure = AttributeError('descriptor failed')
|
||||
class Snapshot:
|
||||
@property
|
||||
def flag(self): raise failure
|
||||
snapshot = Snapshot()
|
||||
class Dynamic:
|
||||
def __getattr__(self, name): raise failure
|
||||
class Intercepted:
|
||||
def __getattribute__(self, name): raise failure
|
||||
dynamic = Dynamic()
|
||||
intercepted = Intercepted()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let snapshot = locals.get_item("snapshot").unwrap().unwrap();
|
||||
let descriptor = PyErr::from(Field::read(&snapshot, "test", "flag").err().unwrap());
|
||||
assert!(
|
||||
descriptor
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
for name in ["dynamic", "intercepted"] {
|
||||
let value = locals.get_item(name).unwrap().unwrap();
|
||||
let error = PyErr::from(Field::read(&value, "test", "flag").err().unwrap());
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
}
|
||||
let missing = PyErr::from(Field::read(&snapshot, "test", "missing").err().unwrap());
|
||||
assert!(missing.is_instance_of::<PyRuntimeError>(py));
|
||||
assert!(missing.to_string().contains("test.missing"));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configuration_errors_name_fields_without_exposing_values() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
for source in [
|
||||
"{'secret': 'do-not-print'}",
|
||||
"['host.test', {'secret': 'do-not-print'}]",
|
||||
] {
|
||||
let field = Field::new("test", "setting", evaluate(py, source));
|
||||
let error = PyErr::from(field.falsy_optional_string().err().unwrap());
|
||||
assert!(error.is_instance_of::<PyValueError>(py));
|
||||
assert!(error.to_string().contains("test.setting"));
|
||||
assert!(!error.to_string().contains("do-not-print"));
|
||||
}
|
||||
let hosts = Field::new(
|
||||
"url_policy",
|
||||
"user_url_allowed_hosts",
|
||||
evaluate(py, "['host.test', 1]"),
|
||||
);
|
||||
assert!(matches!(
|
||||
hosts.string_collection(),
|
||||
Err(ProjectionError::InvalidConfiguration(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
Field::new("test", "flag", evaluate(py, "1")).str_bool(),
|
||||
Err(ProjectionError::InvalidConfiguration(_))
|
||||
));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,372 +0,0 @@
|
|||
use std::ffi::CString;
|
||||
|
||||
use pyo3::{
|
||||
exceptions::{PyLookupError, PyRuntimeError, PyValueError},
|
||||
types::PyDict,
|
||||
};
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
|
||||
py.eval(&CString::new(source).unwrap(), None, None).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("None", false, false)]
|
||||
#[case("False", false, false)]
|
||||
#[case("True", true, true)]
|
||||
#[case("0", false, false)]
|
||||
#[case("1", true, false)]
|
||||
#[case("''", false, false)]
|
||||
#[case("'false'", true, false)]
|
||||
#[case("[]", false, false)]
|
||||
#[case("[0]", true, false)]
|
||||
#[case("{}", false, false)]
|
||||
#[case("object()", true, false)]
|
||||
fn boolean_operations_have_distinct_python_semantics(
|
||||
#[case] source: &str,
|
||||
#[case] truth: bool,
|
||||
#[case] exact: bool,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let value = evaluate(py, source);
|
||||
let field = Field::new("test.flag", value.clone());
|
||||
assert_eq!(field.truthy().unwrap().0, truth);
|
||||
assert_eq!(field.exact_true().0, exact);
|
||||
assert_eq!(
|
||||
field.truthy().unwrap().0,
|
||||
py.import("builtins")
|
||||
.unwrap()
|
||||
.getattr("bool")
|
||||
.unwrap()
|
||||
.call1((value,))
|
||||
.unwrap()
|
||||
.extract::<bool>()
|
||||
.unwrap()
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("None", Ok(None), Ok(None), Ok(None))]
|
||||
#[case("''", Ok(Some("")), Ok(None), Ok(None))]
|
||||
#[case(
|
||||
"' value '",
|
||||
Ok(Some(" value ")),
|
||||
Ok(Some(" value ")),
|
||||
Ok(Some(" value "))
|
||||
)]
|
||||
#[case("[]", Err(()), Ok(None), Ok(None))]
|
||||
#[case("0", Err(()), Ok(None), Ok(None))]
|
||||
#[case("1", Err(()), Err(()), Ok(None))]
|
||||
#[case("object()", Err(()), Err(()), Ok(None))]
|
||||
fn string_operations_do_not_conflate_absence_and_type_checks(
|
||||
#[case] source: &str,
|
||||
#[case] strict: Result<Option<&str>, ()>,
|
||||
#[case] fallback: Result<Option<&str>, ()>,
|
||||
#[case] tuning: Result<Option<&str>, ()>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let field = Field::new("test.string", evaluate(py, source));
|
||||
let owned =
|
||||
|expected: Result<Option<&str>, ()>| expected.map(|value| value.map(str::to_owned));
|
||||
assert_eq!(
|
||||
field
|
||||
.optional_strict_string()
|
||||
.map(|value| value.0)
|
||||
.map_err(|_| ()),
|
||||
owned(strict)
|
||||
);
|
||||
assert_eq!(
|
||||
field
|
||||
.falsy_optional_string()
|
||||
.map(|value| value.0)
|
||||
.map_err(|_| ()),
|
||||
owned(fallback)
|
||||
);
|
||||
assert_eq!(
|
||||
field.tuning_string().map(|value| value.0).map_err(|_| ()),
|
||||
owned(tuning)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("None", None)]
|
||||
#[case("' True '", Some(true))]
|
||||
#[case("' fAlSe '", Some(false))]
|
||||
#[case("'yes'", None)]
|
||||
#[case("'1'", None)]
|
||||
#[case("'unknown'", None)]
|
||||
fn string_boolean_tokens_remain_separate_from_truthiness(
|
||||
#[case] source: &str,
|
||||
#[case] expected: Option<bool>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
assert_eq!(
|
||||
Field::new("test.flag", evaluate(py, source))
|
||||
.str_bool()
|
||||
.unwrap()
|
||||
.0,
|
||||
expected
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("'EXAMPLE.TEST.'", vec!["example.test"])]
|
||||
#[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])]
|
||||
#[case("('B.test', 'a.test')", vec!["a.test", "b.test"])]
|
||||
#[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])]
|
||||
#[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])]
|
||||
#[case("None", vec![])]
|
||||
#[case("False", vec![])]
|
||||
fn host_collection_is_owned_normalized_and_deterministic(
|
||||
#[case] source: &str,
|
||||
#[case] expected: Vec<&str>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
assert_eq!(
|
||||
Field::new("url_policy.user_url_allowed_hosts", evaluate(py, source))
|
||||
.host_collection()
|
||||
.unwrap()
|
||||
.0,
|
||||
expected
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn protocol_errors_preserve_exception_identity_traceback_cause_and_context() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
failure = LookupError('protocol failed')
|
||||
cause = ValueError('cause')
|
||||
context = RuntimeError('context')
|
||||
def fail():
|
||||
try:
|
||||
raise context
|
||||
except RuntimeError:
|
||||
raise failure from cause
|
||||
class Bool:
|
||||
def __bool__(self): return fail()
|
||||
class Length:
|
||||
def __len__(self): return fail()
|
||||
class Iter:
|
||||
def __iter__(self): return fail()
|
||||
class Next:
|
||||
def __iter__(self): return self
|
||||
def __next__(self): return fail()
|
||||
class Descriptor:
|
||||
@property
|
||||
def flag(self): return fail()
|
||||
values = (Bool(), Length(), Iter(), Next(), [Bool()])
|
||||
descriptor = Descriptor()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let values = locals.get_item("values").unwrap().unwrap();
|
||||
for value in values.try_iter().unwrap() {
|
||||
let error = Field::new("test.flag", value.unwrap())
|
||||
.host_collection()
|
||||
.err()
|
||||
.unwrap();
|
||||
let error = PyErr::from(error);
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
assert!(error.is_instance_of::<PyLookupError>(py));
|
||||
assert!(error.traceback(py).is_some());
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.getattr("__cause__")
|
||||
.unwrap()
|
||||
.is(locals.get_item("cause").unwrap().unwrap())
|
||||
);
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.getattr("__context__")
|
||||
.unwrap()
|
||||
.is(locals.get_item("context").unwrap().unwrap())
|
||||
);
|
||||
}
|
||||
let error = Field::read(
|
||||
&locals.get_item("descriptor").unwrap().unwrap(),
|
||||
"test.flag",
|
||||
)
|
||||
.err()
|
||||
.unwrap();
|
||||
assert!(
|
||||
PyErr::from(error)
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_and_string_contents_do_not_invoke_unrelated_protocols() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
class Hostile:
|
||||
def __bool__(self): raise AssertionError('bool called')
|
||||
def __eq__(self, other): raise AssertionError('eq called')
|
||||
def __str__(self): raise AssertionError('str called')
|
||||
class Text(str):
|
||||
def __str__(self): raise AssertionError('str called')
|
||||
def strip(self): raise AssertionError('strip called')
|
||||
def lower(self): raise AssertionError('lower called')
|
||||
hostile = Hostile()
|
||||
text = Text(' False ')
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let hostile = Field::new("test.flag", locals.get_item("hostile").unwrap().unwrap());
|
||||
assert!(!hostile.exact_true().0);
|
||||
assert!(matches!(
|
||||
hostile.strict_string(),
|
||||
Err(ProjectionError::InvalidConfiguration(_))
|
||||
));
|
||||
let text = Field::new("test.flag", locals.get_item("text").unwrap().unwrap());
|
||||
assert_eq!(text.strict_string().unwrap(), " False ");
|
||||
assert_eq!(text.str_bool().unwrap().0, Some(false));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_snapshot_fields_and_descriptor_attribute_errors_are_distinct() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
failure = AttributeError('descriptor failed')
|
||||
class Snapshot:
|
||||
@property
|
||||
def flag(self): raise failure
|
||||
snapshot = Snapshot()
|
||||
class Dynamic:
|
||||
def __getattr__(self, name): raise failure
|
||||
class Intercepted:
|
||||
def __getattribute__(self, name): raise failure
|
||||
dynamic = Dynamic()
|
||||
intercepted = Intercepted()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let snapshot = locals.get_item("snapshot").unwrap().unwrap();
|
||||
let descriptor = PyErr::from(Field::read(&snapshot, "test.flag").err().unwrap());
|
||||
assert!(
|
||||
descriptor
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
for name in ["dynamic", "intercepted"] {
|
||||
let value = locals.get_item(name).unwrap().unwrap();
|
||||
let error = PyErr::from(Field::read(&value, "test.flag").err().unwrap());
|
||||
assert!(
|
||||
error
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
}
|
||||
let missing = PyErr::from(Field::read(&snapshot, "test.missing").err().unwrap());
|
||||
assert!(missing.is_instance_of::<PyRuntimeError>(py));
|
||||
assert!(missing.to_string().contains("test.missing"));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configuration_errors_name_fields_without_exposing_values() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
for source in [
|
||||
"{'secret': 'do-not-print'}",
|
||||
"['host.test', {'secret': 'do-not-print'}]",
|
||||
] {
|
||||
let field = Field::new("test.setting", evaluate(py, source));
|
||||
let error = PyErr::from(field.falsy_optional_string().err().unwrap());
|
||||
assert!(error.is_instance_of::<PyValueError>(py));
|
||||
assert!(error.to_string().contains("test.setting"));
|
||||
assert!(!error.to_string().contains("do-not-print"));
|
||||
}
|
||||
let hosts = Field::new(
|
||||
"url_policy.user_url_allowed_hosts",
|
||||
evaluate(py, "['host.test', 1]"),
|
||||
);
|
||||
assert!(matches!(
|
||||
hosts.host_collection(),
|
||||
Err(ProjectionError::InvalidConfiguration(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
Field::new("test.flag", evaluate(py, "1")).str_bool(),
|
||||
Err(ProjectionError::InvalidConfiguration(_))
|
||||
));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_releases_the_source_collection() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let source = evaluate(py, "['A.test']");
|
||||
let projected = Field::new("test.hosts", source.clone())
|
||||
.host_collection()
|
||||
.unwrap()
|
||||
.0;
|
||||
source.call_method1("append", ("b.test",)).unwrap();
|
||||
assert_eq!(projected, ["a.test"]);
|
||||
assert_eq!(
|
||||
Field::new("test.hosts", source)
|
||||
.host_collection()
|
||||
.unwrap()
|
||||
.0,
|
||||
["a.test", "b.test"]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("True", Some(true))]
|
||||
#[case("False", Some(false))]
|
||||
#[case("1", None)]
|
||||
#[case("None", None)]
|
||||
#[case("[]", None)]
|
||||
fn accessor_booleans_are_strict_schema_values(
|
||||
#[case] source: &str,
|
||||
#[case] expected: Option<bool>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let result = Field::new("secret_manager.readable", evaluate(py, source)).schema_bool();
|
||||
match expected {
|
||||
Some(expected) => assert_eq!(result.unwrap(), expected),
|
||||
None => {
|
||||
let error = PyErr::from(result.unwrap_err());
|
||||
assert!(error.is_instance_of::<PyRuntimeError>(py));
|
||||
assert!(error.to_string().contains("secret_manager.readable"));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
use std::{
|
||||
collections::HashSet,
|
||||
collections::{BTreeSet, HashSet},
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, LazyLock, Mutex, PoisonError},
|
||||
};
|
||||
|
|
@ -10,9 +10,75 @@ use litellm_http::{
|
|||
TlsSource, Unsupported,
|
||||
media::{PublicDnsResolver, UrlPolicy},
|
||||
};
|
||||
use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
|
||||
use pyo3::{
|
||||
exceptions::PyValueError,
|
||||
prelude::*,
|
||||
types::{PyBool, PyDict, PyString},
|
||||
};
|
||||
|
||||
use crate::{coercion::Field, python_settings::PythonSettings};
|
||||
use crate::{
|
||||
coercion::{Field, FieldSpec, ProjectionError},
|
||||
python_settings::{PythonSettings, Snapshot},
|
||||
};
|
||||
|
||||
const SSL_VERIFY: FieldSpec<Option<SslVerify>> = FieldSpec::new("ssl_verify", decode_ssl_verify);
|
||||
const SSL_CERTIFICATE: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("ssl_certificate", |field| field.optional_strict_string());
|
||||
const SSL_SECURITY_LEVEL: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("ssl_security_level", |field| field.tuning_string());
|
||||
const SSL_ECDH_CURVE: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("ssl_ecdh_curve", |field| field.tuning_string());
|
||||
const FORCE_IPV4: FieldSpec<bool> = FieldSpec::new("force_ipv4", |field| field.truthy());
|
||||
const HTTP2: FieldSpec<bool> = FieldSpec::new("http2", |field| Ok(field.exact_true()));
|
||||
const AIOHTTP_TRUST_ENV: FieldSpec<bool> =
|
||||
FieldSpec::new("aiohttp_trust_env", |field| field.truthy());
|
||||
const DISABLE_AIOHTTP_TRUST_ENV: FieldSpec<bool> =
|
||||
FieldSpec::new("disable_aiohttp_trust_env", |field| field.truthy());
|
||||
const DISABLE_AIOHTTP_TRANSPORT: FieldSpec<bool> =
|
||||
FieldSpec::new("disable_aiohttp_transport", |field| Ok(field.exact_true()));
|
||||
const USER_AGENT: FieldSpec<String> = FieldSpec::new("user_agent", |field| field.schema_string());
|
||||
const USER_URL_VALIDATION: FieldSpec<bool> =
|
||||
FieldSpec::new("user_url_validation", |field| field.truthy());
|
||||
const USER_URL_ALLOWED_HOSTS: FieldSpec<Vec<String>> =
|
||||
FieldSpec::new("user_url_allowed_hosts", decode_hosts);
|
||||
|
||||
fn decode_hosts(field: &Field<'_>) -> Result<Vec<String>, ProjectionError> {
|
||||
Ok(field
|
||||
.string_collection()?
|
||||
.into_iter()
|
||||
.map(|host| litellm_http::media::normalize_host(&host))
|
||||
.collect::<BTreeSet<_>>()
|
||||
.into_iter()
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn decode_ssl_verify(field: &Field<'_>) -> Result<Option<SslVerify>, ProjectionError> {
|
||||
let value = field.value();
|
||||
if value.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
if value.is_instance_of::<PyBool>() {
|
||||
return Ok(Some(if field.exact_true() {
|
||||
SslVerify::Enabled
|
||||
} else {
|
||||
SslVerify::Disabled
|
||||
}));
|
||||
}
|
||||
if value.is_instance_of::<PyString>() {
|
||||
return Ok(Some(match field.str_bool()? {
|
||||
Some(true) => SslVerify::Enabled,
|
||||
Some(false) => SslVerify::Disabled,
|
||||
None => SslVerify::CaBundle(field.strict_string()?.into()),
|
||||
}));
|
||||
}
|
||||
let context = value.py().import("ssl")?.getattr("SSLContext")?;
|
||||
if value.is_instance(&context)? {
|
||||
return Err(ProjectionError::UnsupportedLiveObject(field.expected(
|
||||
"a Boolean, Boolean string, CA path, or None; live SSLContext is unsupported",
|
||||
)?));
|
||||
}
|
||||
Err(field.invalid("a Boolean, Boolean string, CA path, or None"))
|
||||
}
|
||||
|
||||
static POOL: LazyLock<HttpClientPool> =
|
||||
LazyLock::new(|| HttpClientPool::new(Arc::new(PublicDnsResolver)));
|
||||
|
|
@ -80,20 +146,20 @@ pub(crate) fn url_policy(py: Python<'_>) -> PyResult<UrlPolicy> {
|
|||
project_url_policy(&PythonSettings::UrlPolicy.read(py)?)
|
||||
}
|
||||
|
||||
fn project_url_policy(value: &Bound<'_, PyAny>) -> PyResult<UrlPolicy> {
|
||||
fn project_url_policy(snapshot: &Snapshot<'_>) -> PyResult<UrlPolicy> {
|
||||
Ok(UrlPolicy {
|
||||
validate: Field::read(value, "url_policy.user_url_validation")?
|
||||
.truthy()?
|
||||
.0,
|
||||
allowed_hosts: Field::read(value, "url_policy.user_url_allowed_hosts")?
|
||||
.host_collection()?
|
||||
.0,
|
||||
validate: snapshot.read(&USER_URL_VALIDATION)?,
|
||||
allowed_hosts: snapshot.read(&USER_URL_ALLOWED_HOSTS)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn call_ssl_verify(kwargs: &Bound<'_, PyDict>) -> PyResult<Option<SslVerify>> {
|
||||
match kwargs.get_item("ssl_verify")? {
|
||||
Some(value) => Ok(Field::new("request.ssl_verify", value).ssl_verify()?.0),
|
||||
Some(value) => Ok(decode_ssl_verify(&Field::new(
|
||||
"request",
|
||||
"ssl_verify",
|
||||
value,
|
||||
))?),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
|
@ -106,39 +172,18 @@ fn for_call(call_ssl_verify: Option<SslVerify>, asynchronous: bool) -> HttpSetti
|
|||
}
|
||||
}
|
||||
|
||||
fn configured(value: &Bound<'_, PyAny>) -> PyResult<HttpSettingsLayer> {
|
||||
fn configured(snapshot: &Snapshot<'_>) -> PyResult<HttpSettingsLayer> {
|
||||
Ok(HttpSettingsLayer {
|
||||
ssl_verify: Field::read(value, "http_settings.ssl_verify")?
|
||||
.ssl_verify()?
|
||||
.0,
|
||||
ssl_certificate: Field::read(value, "http_settings.ssl_certificate")?
|
||||
.optional_strict_string()?
|
||||
.0
|
||||
.map(PathBuf::from),
|
||||
ssl_security_level: Field::read(value, "http_settings.ssl_security_level")?
|
||||
.tuning_string()?
|
||||
.0,
|
||||
ssl_ecdh_curve: Field::read(value, "http_settings.ssl_ecdh_curve")?
|
||||
.tuning_string()?
|
||||
.0,
|
||||
force_ipv4: Some(Field::read(value, "http_settings.force_ipv4")?.truthy()?.0),
|
||||
http2: Some(Field::read(value, "http_settings.http2")?.exact_true().0),
|
||||
aiohttp_trust_env: Some(
|
||||
Field::read(value, "http_settings.aiohttp_trust_env")?
|
||||
.truthy()?
|
||||
.0,
|
||||
),
|
||||
disable_aiohttp_trust_env: Some(
|
||||
Field::read(value, "http_settings.disable_aiohttp_trust_env")?
|
||||
.truthy()?
|
||||
.0,
|
||||
),
|
||||
disable_aiohttp_transport: Some(
|
||||
Field::read(value, "http_settings.disable_aiohttp_transport")?
|
||||
.exact_true()
|
||||
.0,
|
||||
),
|
||||
user_agent: Some(Field::read(value, "http_settings.user_agent")?.schema_string()?),
|
||||
ssl_verify: snapshot.read(&SSL_VERIFY)?,
|
||||
ssl_certificate: snapshot.read(&SSL_CERTIFICATE)?.map(PathBuf::from),
|
||||
ssl_security_level: snapshot.read(&SSL_SECURITY_LEVEL)?,
|
||||
ssl_ecdh_curve: snapshot.read(&SSL_ECDH_CURVE)?,
|
||||
force_ipv4: Some(snapshot.read(&FORCE_IPV4)?),
|
||||
http2: Some(snapshot.read(&HTTP2)?),
|
||||
aiohttp_trust_env: Some(snapshot.read(&AIOHTTP_TRUST_ENV)?),
|
||||
disable_aiohttp_trust_env: Some(snapshot.read(&DISABLE_AIOHTTP_TRUST_ENV)?),
|
||||
disable_aiohttp_transport: Some(snapshot.read(&DISABLE_AIOHTTP_TRANSPORT)?),
|
||||
user_agent: Some(snapshot.read(&USER_AGENT)?),
|
||||
..HttpSettingsLayer::default()
|
||||
})
|
||||
}
|
||||
|
|
@ -150,12 +195,15 @@ mod tests {
|
|||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
use crate::python_settings::CONTRACT;
|
||||
|
||||
fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Bound<'py, PyAny> {
|
||||
fn evaluate<'py>(py: Python<'py>, source: &str) -> Bound<'py, PyAny> {
|
||||
py.eval(&std::ffi::CString::new(source).unwrap(), None, None)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn python_settings<'py>(py: Python<'py>, overrides: &str) -> Snapshot<'py> {
|
||||
let source = format!(
|
||||
"
|
||||
import json
|
||||
import types
|
||||
defaults = dict(
|
||||
ssl_verify=True,
|
||||
|
|
@ -170,14 +218,13 @@ defaults = dict(
|
|||
user_agent='litellm/test',
|
||||
)
|
||||
defaults.update(dict({overrides}))
|
||||
settings = types.SimpleNamespace(**{{name: defaults[name] for name in json.loads(contract)['http_settings']['fields']}})
|
||||
settings = types.SimpleNamespace(**defaults)
|
||||
"
|
||||
);
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("contract", CONTRACT).unwrap();
|
||||
let source = std::ffi::CString::new(source).unwrap();
|
||||
py.run(&source, Some(&locals), Some(&locals)).unwrap();
|
||||
locals.get_item("settings").unwrap().unwrap()
|
||||
PythonSettings::Http.snapshot(locals.get_item("settings").unwrap().unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -395,7 +442,7 @@ user_agent='litellm/9.9.9',
|
|||
Python::attach(|py| {
|
||||
let value = py.eval(c"__import__('types').SimpleNamespace(user_url_validation=[], user_url_allowed_hosts=['B.test', 'a.test.', 'b.test'])", None, None).unwrap();
|
||||
assert_eq!(
|
||||
project_url_policy(&value).unwrap(),
|
||||
project_url_policy(&PythonSettings::UrlPolicy.snapshot(value)).unwrap(),
|
||||
UrlPolicy {
|
||||
validate: false,
|
||||
allowed_hosts: vec!["a.test".into(), "b.test".into()],
|
||||
|
|
@ -419,4 +466,44 @@ user_agent='litellm/9.9.9',
|
|||
let settings = HttpSettings::from_layers([for_call(None, asynchronous), opted_out]);
|
||||
assert_eq!(settings.trust_proxy_env, expected);
|
||||
}
|
||||
#[rstest]
|
||||
#[case("'EXAMPLE.TEST.'", vec!["example.test"])]
|
||||
#[case("['B.test', '', None, 0, [], 'A.test.', 'b.test']", vec!["a.test", "b.test"])]
|
||||
#[case("('B.test', 'a.test')", vec!["a.test", "b.test"])]
|
||||
#[case("{'B.test', 'a.test'}", vec!["a.test", "b.test"])]
|
||||
#[case("(host for host in ['B.test', 'a.test'])", vec!["a.test", "b.test"])]
|
||||
#[case("None", vec![])]
|
||||
#[case("False", vec![])]
|
||||
fn host_collection_is_owned_normalized_and_deterministic(
|
||||
#[case] source: &str,
|
||||
#[case] expected: Vec<&str>,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
assert_eq!(
|
||||
decode_hosts(&Field::new(
|
||||
"url_policy",
|
||||
"user_url_allowed_hosts",
|
||||
evaluate(py, source)
|
||||
))
|
||||
.unwrap(),
|
||||
expected
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_releases_the_source_collection() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let source = evaluate(py, "['A.test']");
|
||||
let projected = decode_hosts(&Field::new("test", "hosts", source.clone())).unwrap();
|
||||
source.call_method1("append", ("b.test",)).unwrap();
|
||||
assert_eq!(projected, ["a.test"]);
|
||||
assert_eq!(
|
||||
decode_hosts(&Field::new("test", "hosts", source)).unwrap(),
|
||||
["a.test", "b.test"]
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,12 @@ mod http;
|
|||
mod marshal;
|
||||
mod python_settings;
|
||||
mod routes;
|
||||
mod token_counter;
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "secret-manager foundations await rollout activation"
|
||||
)]
|
||||
mod secrets;
|
||||
mod tokenizer;
|
||||
|
||||
#[pymodule(gil_used = true)]
|
||||
mod _native {
|
||||
|
|
@ -32,7 +37,12 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::routes::responses::ResponsesWebSocketConnection;
|
||||
#[pymodule_export]
|
||||
use crate::token_counter::TokenCounter;
|
||||
use crate::routes::token_counter::TokenCounter;
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pymodule_export]
|
||||
use crate::tokenizer::HuggingFaceEncoding;
|
||||
#[pymodule_export]
|
||||
use crate::tokenizer::Tokenizer;
|
||||
#[pymodule_export]
|
||||
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
|
||||
use pyo3::{prelude::*, types::PyModule};
|
||||
|
|
@ -43,7 +53,7 @@ mod _native {
|
|||
let dict = module.dict();
|
||||
dict.set_item("_CacheTestHandle", py.get_type::<CacheTestHandle>())?;
|
||||
dict.set_item("_CacheTestResolver", py.get_type::<CacheTestResolver>())?;
|
||||
dict.set_item("_CacheTestBinding", py.get_type::<ResolvedCache>())
|
||||
dict.set_item("_ResponseCacheRuntime", py.get_type::<ResolvedCache>())
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -78,10 +88,13 @@ mod tests {
|
|||
"achat_completions",
|
||||
"ResponsesWebSocketConnection",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"gil_stats",
|
||||
"process_state_started",
|
||||
"reserve_process_for_forking",
|
||||
];
|
||||
#[cfg(feature = "huggingface")]
|
||||
expected.push("HuggingFaceEncoding");
|
||||
expected.sort_unstable();
|
||||
|
||||
let mut public_names: Vec<String> = native_module(py)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
use pyo3::prelude::*;
|
||||
|
||||
use crate::coercion::{FieldSpec, ProjectionError};
|
||||
|
||||
const MODULE: &str = "litellm.rust_bridge.settings";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
|
|
@ -8,28 +10,39 @@ pub(crate) enum PythonSettings {
|
|||
UrlPolicy,
|
||||
ProviderDefaults,
|
||||
SecretManager,
|
||||
SecretManagerBinding,
|
||||
}
|
||||
|
||||
pub(crate) struct Snapshot<'py> {
|
||||
group: PythonSettings,
|
||||
value: Bound<'py, PyAny>,
|
||||
}
|
||||
|
||||
impl Snapshot<'_> {
|
||||
pub(crate) fn read<T>(&self, spec: &FieldSpec<T>) -> Result<T, ProjectionError> {
|
||||
spec.read(&self.value, self.group.name())
|
||||
}
|
||||
}
|
||||
|
||||
impl PythonSettings {
|
||||
#[cfg(test)]
|
||||
pub(crate) const ALL: [Self; 4] = [
|
||||
Self::Http,
|
||||
Self::UrlPolicy,
|
||||
Self::ProviderDefaults,
|
||||
Self::SecretManager,
|
||||
];
|
||||
|
||||
pub(crate) fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Http => "http_settings",
|
||||
Self::UrlPolicy => "url_policy",
|
||||
Self::ProviderDefaults => "provider_defaults",
|
||||
Self::SecretManager => "secret_manager",
|
||||
Self::SecretManagerBinding => "secret_manager_binding",
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn read(self, py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
py.import(MODULE)?.getattr(self.name())?.call0()
|
||||
pub(crate) fn read(self, py: Python<'_>) -> PyResult<Snapshot<'_>> {
|
||||
let value = py.import(MODULE)?.getattr(self.name())?.call0()?;
|
||||
Ok(Snapshot { group: self, value })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> {
|
||||
Snapshot { group: self, value }
|
||||
}
|
||||
|
||||
pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> {
|
||||
|
|
@ -38,209 +51,98 @@ impl PythonSettings {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) const CONTRACT: &str = include_str!("../python_settings.json");
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CONTRACT, PythonSettings};
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Value, json};
|
||||
use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict};
|
||||
|
||||
struct SettingSpec {
|
||||
group: &'static str,
|
||||
name: &'static str,
|
||||
adapter: &'static str,
|
||||
precedence: &'static str,
|
||||
sensitive: bool,
|
||||
shapes: &'static [&'static str],
|
||||
unsupported_live: Option<&'static str>,
|
||||
}
|
||||
|
||||
const SETTINGS: &[SettingSpec] = &[
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "ssl_verify",
|
||||
adapter: "SslVerifyInput",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &["none", "bool", "str"],
|
||||
unsupported_live: Some("configuration_error"),
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "ssl_certificate",
|
||||
adapter: "OptionalStrictString",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "ssl_security_level",
|
||||
adapter: "TuningString",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "ssl_ecdh_curve",
|
||||
adapter: "TuningString",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "force_ipv4",
|
||||
adapter: "Truthy",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "http2",
|
||||
adapter: "ExactTrue",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "aiohttp_trust_env",
|
||||
adapter: "Truthy",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "disable_aiohttp_trust_env",
|
||||
adapter: "Truthy",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "disable_aiohttp_transport",
|
||||
adapter: "ExactTrue",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "http_settings",
|
||||
name: "user_agent",
|
||||
adapter: "StrictString",
|
||||
precedence: "accessor",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "url_policy",
|
||||
name: "user_url_validation",
|
||||
adapter: "Truthy",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "url_policy",
|
||||
name: "user_url_allowed_hosts",
|
||||
adapter: "HostCollection",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "provider_defaults",
|
||||
name: "vertex_project",
|
||||
adapter: "FalsyOptionalString",
|
||||
precedence: "module_global",
|
||||
sensitive: true,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "provider_defaults",
|
||||
name: "vertex_location",
|
||||
adapter: "FalsyOptionalString",
|
||||
precedence: "module_global",
|
||||
sensitive: true,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "provider_defaults",
|
||||
name: "enable_azure_ad_token_refresh",
|
||||
adapter: "ExactTrue",
|
||||
precedence: "module_global",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
SettingSpec {
|
||||
group: "secret_manager",
|
||||
name: "readable",
|
||||
adapter: "StrictBool",
|
||||
precedence: "accessor",
|
||||
sensitive: false,
|
||||
shapes: &[],
|
||||
unsupported_live: None,
|
||||
},
|
||||
];
|
||||
use super::PythonSettings;
|
||||
use crate::coercion::FieldSpec;
|
||||
|
||||
#[test]
|
||||
fn settings_manifest_matches_the_semantic_contract() {
|
||||
pyo3::Python::initialize();
|
||||
let manifest: Value = pyo3::Python::attach(|py| {
|
||||
let value = py
|
||||
.import("json")
|
||||
.unwrap()
|
||||
.call_method1("loads", (CONTRACT,))
|
||||
.unwrap();
|
||||
litellm_host_python::from_py(&value).unwrap()
|
||||
fn declarations_select_the_decoder_and_read_only_the_requested_field() {
|
||||
const TRUTHY: FieldSpec<bool> = FieldSpec::new("flag", |field| field.truthy());
|
||||
const EXACT: FieldSpec<bool> = FieldSpec::new("flag", |field| Ok(field.exact_true()));
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
reads = []
|
||||
class Settings:
|
||||
value = 1
|
||||
@property
|
||||
def flag(self):
|
||||
reads.append('flag')
|
||||
return self.value
|
||||
@property
|
||||
def unrelated(self):
|
||||
raise AssertionError('unrequested field')
|
||||
settings = Settings()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let value = locals.get_item("settings").unwrap().unwrap();
|
||||
let snapshot = PythonSettings::Http.snapshot(value.clone());
|
||||
assert!(snapshot.read(&TRUTHY).unwrap());
|
||||
assert!(!snapshot.read(&EXACT).unwrap());
|
||||
value.setattr("value", true).unwrap();
|
||||
assert!(snapshot.read(&EXACT).unwrap());
|
||||
assert_eq!(
|
||||
locals
|
||||
.get_item("reads")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract::<Vec<String>>()
|
||||
.unwrap(),
|
||||
["flag", "flag", "flag"]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn declared_reads_preserve_descriptor_and_decoder_failures_and_name_missing_fields() {
|
||||
const FLAG: FieldSpec<bool> = FieldSpec::new("flag", |field| field.truthy());
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
from types import SimpleNamespace
|
||||
failure = AttributeError('read failed')
|
||||
class Descriptor:
|
||||
@property
|
||||
def flag(self): raise failure
|
||||
class Truth:
|
||||
def __bool__(self): raise failure
|
||||
values = (Descriptor(), SimpleNamespace(flag=Truth()))
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let failure = locals.get_item("failure").unwrap().unwrap();
|
||||
for value in locals
|
||||
.get_item("values")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.try_iter()
|
||||
.unwrap()
|
||||
{
|
||||
let snapshot = PythonSettings::Http.snapshot(value.unwrap());
|
||||
let error = PyErr::from(snapshot.read(&FLAG).unwrap_err());
|
||||
assert!(error.value(py).is(&failure));
|
||||
assert!(error.traceback(py).is_some());
|
||||
}
|
||||
let missing = PythonSettings::Http.snapshot(py.eval(c"object()", None, None).unwrap());
|
||||
let error = PyErr::from(missing.read(&FLAG).unwrap_err());
|
||||
assert!(error.is_instance_of::<PyRuntimeError>(py));
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("http_settings.flag: missing snapshot field")
|
||||
);
|
||||
});
|
||||
let expected: serde_json::Map<String, Value> = PythonSettings::ALL
|
||||
.into_iter()
|
||||
.map(|group| {
|
||||
let fields: serde_json::Map<String, Value> = SETTINGS
|
||||
.iter()
|
||||
.filter(|spec| spec.group == group.name())
|
||||
.map(|spec| {
|
||||
(
|
||||
spec.name.to_owned(),
|
||||
json!({
|
||||
"adapter": spec.adapter,
|
||||
"required": true,
|
||||
"precedence": spec.precedence,
|
||||
"sensitive": spec.sensitive,
|
||||
"shapes": spec.shapes,
|
||||
"unsupported_live": spec.unsupported_live,
|
||||
}),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
(
|
||||
group.name().to_owned(),
|
||||
json!({"version": 1, "fields": fields}),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(manifest, Value::Object(expected));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ pub(crate) mod chat_completions;
|
|||
pub(crate) mod messages;
|
||||
pub(crate) mod ocr;
|
||||
pub(crate) mod responses;
|
||||
pub(crate) mod token_counter;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
|
|
|||
|
|
@ -130,6 +130,11 @@ impl RouteHost for OcrRouteHost {
|
|||
}
|
||||
|
||||
fn classify(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
|
||||
if let Error::Secret(source) = &error
|
||||
&& let Some(original) = crate::secrets::callback::python_error(py, source)
|
||||
{
|
||||
return Ok(original);
|
||||
}
|
||||
Ok(self.map_failure(py, ocr_error_to_pyerr(error)))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,16 +10,33 @@ use litellm_auth_gcp::VertexAuth;
|
|||
use litellm_callbacks_legacy_python::{LegacySurface, PublicCall, run_legacy_call};
|
||||
use litellm_core::ocr::route::ocr_machine;
|
||||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_llms::base_llm::ocr::{
|
||||
handler::OcrClient,
|
||||
settings::{OcrSettings, Secrets},
|
||||
use litellm_llms::base_llm::{
|
||||
inference::secrets::{EnvironmentSecrets, SecretSource},
|
||||
ocr::{handler::OcrClient, settings::OcrSettings},
|
||||
};
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use crate::{coercion::Field, errors::RustBridgeDeclined, http, python_settings::PythonSettings};
|
||||
use crate::{
|
||||
coercion::FieldSpec,
|
||||
errors::RustBridgeDeclined,
|
||||
http,
|
||||
python_settings::{PythonSettings, Snapshot},
|
||||
};
|
||||
|
||||
const SECRET_MANAGER_READABLE: FieldSpec<bool> =
|
||||
FieldSpec::new("readable", |field| field.schema_bool());
|
||||
|
||||
const VERTEX_PROJECT: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("vertex_project", |field| field.falsy_optional_string());
|
||||
const VERTEX_LOCATION: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("vertex_location", |field| field.falsy_optional_string());
|
||||
const ENABLE_AZURE_AD_TOKEN_REFRESH: FieldSpec<bool> =
|
||||
FieldSpec::new("enable_azure_ad_token_refresh", |field| {
|
||||
Ok(field.exact_true())
|
||||
});
|
||||
|
||||
const SURFACE: LegacySurface = LegacySurface {
|
||||
call_type: "ocr",
|
||||
|
|
@ -62,33 +79,24 @@ fn run_ocr(
|
|||
)
|
||||
}
|
||||
|
||||
fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult<Secrets> {
|
||||
if Field::read(secret_manager, "secret_manager.readable")?.schema_bool()? {
|
||||
fn process_environment_secrets(snapshot: &Snapshot<'_>) -> PyResult<Arc<dyn SecretSource>> {
|
||||
if snapshot.read(&SECRET_MANAGER_READABLE)? {
|
||||
return Err(RustBridgeDeclined::new_err(
|
||||
"a readable secret manager is configured and the Rust route only reads the process environment",
|
||||
));
|
||||
}
|
||||
Ok(Arc::new(ProcessEnvironment))
|
||||
Ok(Arc::new(EnvironmentSecrets))
|
||||
}
|
||||
|
||||
fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
|
||||
project_provider_defaults(&PythonSettings::ProviderDefaults.read(py)?)
|
||||
}
|
||||
|
||||
fn project_provider_defaults(value: &Bound<'_, PyAny>) -> PyResult<OcrSettings> {
|
||||
fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult<OcrSettings> {
|
||||
Ok(OcrSettings {
|
||||
vertex_project: Field::read(value, "provider_defaults.vertex_project")?
|
||||
.falsy_optional_string()?
|
||||
.0,
|
||||
vertex_location: Field::read(value, "provider_defaults.vertex_location")?
|
||||
.falsy_optional_string()?
|
||||
.0,
|
||||
enable_azure_ad_token_refresh: Field::read(
|
||||
value,
|
||||
"provider_defaults.enable_azure_ad_token_refresh",
|
||||
)?
|
||||
.exact_true()
|
||||
.0,
|
||||
vertex_project: snapshot.read(&VERTEX_PROJECT)?,
|
||||
vertex_location: snapshot.read(&VERTEX_LOCATION)?,
|
||||
enable_azure_ad_token_refresh: snapshot.read(&ENABLE_AZURE_AD_TOKEN_REFRESH)?,
|
||||
..OcrSettings::from_environment(&ProcessEnvironment)
|
||||
})
|
||||
}
|
||||
|
|
@ -120,6 +128,8 @@ mod tests {
|
|||
use super::process_environment_secrets;
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
use crate::python_settings::PythonSettings;
|
||||
|
||||
fn secret_manager<'py>(py: Python<'py>, readable: bool) -> Bound<'py, PyAny> {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("readable", readable).unwrap();
|
||||
|
|
@ -132,12 +142,26 @@ mod tests {
|
|||
locals.get_item("manager").unwrap().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_readable_secret_manager_sends_the_call_back_to_python() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let declined = process_environment_secrets(
|
||||
&PythonSettings::SecretManager.snapshot(secret_manager(py, true)),
|
||||
)
|
||||
.err()
|
||||
.expect("the Rust route declines");
|
||||
assert!(declined.is_instance_of::<RustBridgeDeclined>(py));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_defaults_distinguish_falsey_values_and_exact_true() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let value = py.eval(c"__import__('types').SimpleNamespace(vertex_project=[], vertex_location=0, enable_azure_ad_token_refresh=1)", None, None).unwrap();
|
||||
let projected = super::project_provider_defaults(&value).unwrap();
|
||||
let snapshot = PythonSettings::ProviderDefaults.snapshot(value.clone());
|
||||
let projected = super::project_provider_defaults(&snapshot).unwrap();
|
||||
assert_eq!(projected.vertex_project, None);
|
||||
assert_eq!(projected.vertex_location, None);
|
||||
assert!(!projected.enable_azure_ad_token_refresh);
|
||||
|
|
@ -146,12 +170,12 @@ mod tests {
|
|||
value
|
||||
.setattr("enable_azure_ad_token_refresh", true)
|
||||
.unwrap();
|
||||
let next = super::project_provider_defaults(&value).unwrap();
|
||||
let next = super::project_provider_defaults(&snapshot).unwrap();
|
||||
assert_eq!(next.vertex_project.as_deref(), Some("project"));
|
||||
assert_eq!(next.vertex_location.as_deref(), Some("region"));
|
||||
assert!(next.enable_azure_ad_token_refresh);
|
||||
value.setattr("vertex_project", 1).unwrap();
|
||||
let error = super::project_provider_defaults(&value).err().unwrap();
|
||||
let error = super::project_provider_defaults(&snapshot).err().unwrap();
|
||||
assert!(error.is_instance_of::<pyo3::exceptions::PyValueError>(py));
|
||||
assert!(
|
||||
error
|
||||
|
|
@ -160,28 +184,4 @@ mod tests {
|
|||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_readable_secret_manager_sends_the_call_back_to_python() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let declined = process_environment_secrets(&secret_manager(py, true))
|
||||
.err()
|
||||
.expect("the Rust route declines");
|
||||
assert!(declined.is_instance_of::<RustBridgeDeclined>(py));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_readable_secret_manager_secrets_are_the_process_environment() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let secrets = process_environment_secrets(&secret_manager(py, false)).unwrap();
|
||||
assert_eq!(
|
||||
secrets.get("LITELLM_RUST_BRIDGE_UNSET_VARIABLE_FOR_TEST"),
|
||||
None
|
||||
);
|
||||
assert_eq!(secrets.get("PATH"), std::env::var("PATH").ok());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
use std::sync::Arc;
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
use litellm_host_python::{enter_native, run_async};
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
types::PyAny,
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::tokenizer::Tokenizer;
|
||||
|
||||
/// Counts the input tokens of a raw request body off the Python event loop with
|
||||
/// the GIL released. Python owns which requests get here and what to do with
|
||||
/// the count. At most one encode per core runs at a time; the rest wait in the
|
||||
/// async task, where a cancelled Python awaiter drops them before any blocking
|
||||
/// work is scheduled.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct TokenCounter {
|
||||
inner: Arc<CoreTokenCounter>,
|
||||
encode_slots: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl TokenCounter {
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (tokenizer, fast = false))]
|
||||
fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult<Self> {
|
||||
enter_native()?;
|
||||
let inner = CoreTokenCounter::new(tokenizer.counter(py, fast));
|
||||
Ok(Self {
|
||||
inner: Arc::new(inner),
|
||||
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
|
||||
})
|
||||
}
|
||||
|
||||
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
|
||||
let counter = Arc::clone(&self.inner);
|
||||
let encode_slots = Arc::clone(&self.encode_slots);
|
||||
let body = body.to_vec();
|
||||
run_async(
|
||||
py,
|
||||
async move {
|
||||
let _slot = encode_slots
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?;
|
||||
tokio::task::spawn_blocking(move || count_body(&counter, &body))
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?
|
||||
},
|
||||
token_count_error_to_pyerr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_parallelism() -> usize {
|
||||
available_parallelism().map_or(1, NonZero::get)
|
||||
}
|
||||
|
||||
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
|
||||
let request = CountableRequest::parse(body)?;
|
||||
counter.count_request(&request)
|
||||
}
|
||||
|
||||
pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr {
|
||||
let message = error.to_string();
|
||||
match error {
|
||||
Error::Load(_)
|
||||
| Error::Ranks(_)
|
||||
| Error::UnicodeClasses
|
||||
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
|
||||
Error::RequestParse(_)
|
||||
| Error::MissingInput
|
||||
| Error::FloatText
|
||||
| Error::ContentBlock
|
||||
| Error::ArrayItems
|
||||
| Error::JsonSerialization(_)
|
||||
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
|
||||
Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
|
||||
}
|
||||
}
|
||||
349
litellm-rust/crates/python-bridge/src/secrets/callback.rs
Normal file
349
litellm-rust/crates/python-bridge/src/secrets/callback.rs
Normal file
|
|
@ -0,0 +1,349 @@
|
|||
use std::{fmt, future::Future, pin::Pin};
|
||||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secrets::{
|
||||
Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
|
||||
};
|
||||
use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
|
||||
|
||||
const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler";
|
||||
|
||||
struct PythonSecretError(Py<PyBaseException>);
|
||||
|
||||
impl fmt::Debug for PythonSecretError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("PythonSecretError")
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PythonSecretError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("Python secret manager failed")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PythonSecretError {}
|
||||
|
||||
pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option<PyErr> {
|
||||
let Error::ExternalManager(source) = error else {
|
||||
return None;
|
||||
};
|
||||
source
|
||||
.downcast_ref::<PythonSecretError>()
|
||||
.map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any()))
|
||||
}
|
||||
|
||||
/// A secret manager whose reads execute in Python: a custom manager, a legacy compatible
|
||||
/// client, or a manually assigned SDK client.
|
||||
pub(crate) struct PythonSecretManager {
|
||||
client: Py<PyAny>,
|
||||
system: Option<KeyManagementSystem>,
|
||||
/// The `key_manager` name Python's handler dispatches on.
|
||||
key_manager: &'static str,
|
||||
settings: Option<Py<PyAny>>,
|
||||
}
|
||||
|
||||
impl PythonSecretManager {
|
||||
pub(crate) fn new(
|
||||
client: Py<PyAny>,
|
||||
system: Option<KeyManagementSystem>,
|
||||
settings: Option<Py<PyAny>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
client,
|
||||
system,
|
||||
key_manager: system.map_or("local", python_name),
|
||||
settings,
|
||||
}
|
||||
}
|
||||
|
||||
fn read(&self, py: Python<'_>, name: &str) -> PyResult<Option<String>> {
|
||||
let client = self.client.bind(py);
|
||||
if self.system == Some(KeyManagementSystem::Custom)
|
||||
|| (self.system.is_none() && client.hasattr("sync_read_secret")?)
|
||||
{
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("secret_name", name)?;
|
||||
if self.system == Some(KeyManagementSystem::Custom) {
|
||||
let optional_params = self
|
||||
.settings
|
||||
.as_ref()
|
||||
.map(|settings| settings.bind(py).call_method0("model_dump"))
|
||||
.transpose()?;
|
||||
kwargs.set_item("optional_params", optional_params)?;
|
||||
}
|
||||
return client
|
||||
.call_method("sync_read_secret", (), Some(&kwargs))?
|
||||
.extract();
|
||||
}
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("client", client)?;
|
||||
kwargs.set_item("key_manager", self.key_manager)?;
|
||||
kwargs.set_item("secret_name", name)?;
|
||||
kwargs.set_item(
|
||||
"key_management_settings",
|
||||
self.settings
|
||||
.as_ref()
|
||||
.map_or_else(|| py.None(), |settings| settings.clone_ref(py)),
|
||||
)?;
|
||||
py.import(HANDLER_MODULE)?
|
||||
.getattr("get_secret_from_manager")?
|
||||
.call((), Some(&kwargs))?
|
||||
.extract()
|
||||
}
|
||||
}
|
||||
|
||||
/// The `KeyManagementSystem` value as Python spells it.
|
||||
fn python_name(system: KeyManagementSystem) -> &'static str {
|
||||
match system {
|
||||
KeyManagementSystem::GoogleKms => "google_kms",
|
||||
KeyManagementSystem::AzureKeyVault => "azure_key_vault",
|
||||
KeyManagementSystem::AwsSecretManager => "aws_secret_manager",
|
||||
KeyManagementSystem::GoogleSecretManager => "google_secret_manager",
|
||||
KeyManagementSystem::HashicorpVault => "hashicorp_vault",
|
||||
KeyManagementSystem::Cyberark => "cyberark",
|
||||
KeyManagementSystem::Local => "local",
|
||||
KeyManagementSystem::AwsKms => "aws_kms",
|
||||
KeyManagementSystem::Custom => "custom",
|
||||
}
|
||||
}
|
||||
|
||||
impl ExternalSecretManager for PythonSecretManager {
|
||||
fn system(&self) -> KeyManagementSystem {
|
||||
self.system.unwrap_or(KeyManagementSystem::Custom)
|
||||
}
|
||||
|
||||
fn read_secret<'a>(
|
||||
&'a self,
|
||||
name: &'a str,
|
||||
_settings: &'a KeyManagementSettings,
|
||||
_environment: &'a (dyn Lookup + Send + Sync),
|
||||
) -> Pin<Box<dyn Future<Output = Result<Option<Secret>, Error>> + Send + 'a>> {
|
||||
Box::pin(async move {
|
||||
Python::attach(|py| {
|
||||
self.read(py, name)
|
||||
.map(|value| value.map(SecretValue::new).map(Secret::String))
|
||||
.map_err(|error| {
|
||||
Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py))))
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use litellm_secrets::{
|
||||
FailurePolicy, KeyManagementSettings, KeyManagementSystem, OidcResolver, SecretManager,
|
||||
SecretManagerState, SecretResolver,
|
||||
};
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
use super::{HANDLER_MODULE, PythonSecretManager, python_error, python_name};
|
||||
|
||||
#[tokio::test]
|
||||
async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback() {
|
||||
Python::initialize();
|
||||
for failure_type in ["ValueError", "asyncio.CancelledError"] {
|
||||
for fallback in [None, Some("environment-key")] {
|
||||
let (reader, locals) = Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("failure_type", failure_type).unwrap();
|
||||
py.run(
|
||||
c"
|
||||
import asyncio
|
||||
failure = eval(failure_type)('secret manager failed')
|
||||
cause = RuntimeError('original cause')
|
||||
context = RuntimeError('original context')
|
||||
failure.__cause__ = cause
|
||||
failure.__context__ = context
|
||||
class Manager:
|
||||
def sync_read_secret(self, secret_name):
|
||||
raise failure
|
||||
manager = Manager()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let reader = PythonSecretManager::new(
|
||||
locals.get_item("manager").unwrap().unwrap().unbind(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
(reader, locals.unbind())
|
||||
});
|
||||
let resolver = SecretResolver::new(
|
||||
Arc::new(SecretManagerState::new(
|
||||
SecretManager::External(Arc::new(reader)),
|
||||
KeyManagementSettings::default(),
|
||||
)),
|
||||
Arc::new(move |_: &str| fallback.map(str::to_owned)),
|
||||
OidcResolver::default(),
|
||||
)
|
||||
.with_failure_policy(FailurePolicy::EnvironmentFallback);
|
||||
let error = resolver.get_secret("API_KEY", None).await.unwrap_err();
|
||||
Python::attach(|py| {
|
||||
let original = python_error(py, &error).unwrap();
|
||||
let locals = locals.bind(py);
|
||||
assert!(
|
||||
original
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] {
|
||||
assert!(
|
||||
original
|
||||
.value(py)
|
||||
.getattr(attribute)
|
||||
.unwrap()
|
||||
.is(locals.get_item(name).unwrap().unwrap())
|
||||
);
|
||||
}
|
||||
assert!(original.traceback(py).is_some());
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and
|
||||
/// removes the fake modules again.
|
||||
fn with_fake_handler<'py>(py: Python<'py>, body: impl FnOnce(&Bound<'py, PyDict>)) {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
import sys, types
|
||||
calls = []
|
||||
def get_secret_from_manager(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return 'handled-' + kwargs['secret_name']
|
||||
handler = types.ModuleType('litellm.secret_managers.secret_manager_handler')
|
||||
handler.get_secret_from_manager = get_secret_from_manager
|
||||
installed = {}
|
||||
for name in ('litellm', 'litellm.secret_managers'):
|
||||
if name not in sys.modules:
|
||||
sys.modules[name] = types.ModuleType(name)
|
||||
installed[name] = True
|
||||
sys.modules['litellm.secret_managers.secret_manager_handler'] = handler
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
body(&locals);
|
||||
py.run(
|
||||
c"
|
||||
sys.modules.pop('litellm.secret_managers.secret_manager_handler', None)
|
||||
for name in installed:
|
||||
sys.modules.pop(name, None)
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn python_names_round_trip_through_serde() {
|
||||
for system in [
|
||||
KeyManagementSystem::GoogleKms,
|
||||
KeyManagementSystem::AzureKeyVault,
|
||||
KeyManagementSystem::AwsSecretManager,
|
||||
KeyManagementSystem::GoogleSecretManager,
|
||||
KeyManagementSystem::HashicorpVault,
|
||||
KeyManagementSystem::Cyberark,
|
||||
KeyManagementSystem::Local,
|
||||
KeyManagementSystem::AwsKms,
|
||||
KeyManagementSystem::Custom,
|
||||
] {
|
||||
assert_eq!(
|
||||
serde_json::to_value(system).unwrap(),
|
||||
serde_json::Value::String(python_name(system).to_owned())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_readers_without_a_system_are_called_directly() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
class Manager:
|
||||
def __init__(self):
|
||||
self.names = []
|
||||
def sync_read_secret(self, secret_name, optional_params=None, timeout=None):
|
||||
self.names.append(secret_name)
|
||||
return 'direct-' + secret_name
|
||||
manager = Manager()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let manager = locals.get_item("manager").unwrap().unwrap();
|
||||
let reader = PythonSecretManager::new(manager.clone().unbind(), None, None);
|
||||
assert_eq!(
|
||||
reader.read(py, "API_KEY").unwrap().as_deref(),
|
||||
Some("direct-API_KEY")
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.getattr("names")
|
||||
.unwrap()
|
||||
.extract::<Vec<String>>()
|
||||
.unwrap(),
|
||||
["API_KEY"]
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configured_systems_dispatch_through_the_python_handler_with_the_original_settings() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
with_fake_handler(py, |locals| {
|
||||
let client = py.eval(c"object()", None, None).unwrap();
|
||||
let settings = py.eval(c"object()", None, None).unwrap();
|
||||
let reader = PythonSecretManager::new(
|
||||
client.clone().unbind(),
|
||||
Some(KeyManagementSystem::AzureKeyVault),
|
||||
Some(settings.clone().unbind()),
|
||||
);
|
||||
assert_eq!(
|
||||
reader.read(py, "API_KEY").unwrap().as_deref(),
|
||||
Some("handled-API_KEY")
|
||||
);
|
||||
assert!(py.import(HANDLER_MODULE).is_ok());
|
||||
let calls = locals.get_item("calls").unwrap().unwrap();
|
||||
let call = calls.get_item(0).unwrap().cast_into::<PyDict>().unwrap();
|
||||
assert!(call.get_item("client").unwrap().unwrap().is(&client));
|
||||
assert!(
|
||||
call.get_item("key_management_settings")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.is(&settings)
|
||||
);
|
||||
assert_eq!(
|
||||
call.get_item("key_manager")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract::<String>()
|
||||
.unwrap(),
|
||||
"azure_key_vault"
|
||||
);
|
||||
assert_eq!(
|
||||
call.get_item("secret_name")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract::<String>()
|
||||
.unwrap(),
|
||||
"API_KEY"
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
338
litellm-rust/crates/python-bridge/src/secrets/config.rs
Normal file
338
litellm-rust/crates/python-bridge/src/secrets/config.rs
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use litellm_secrets::{SecretManager, SecretManagerState};
|
||||
use litellm_secrets_types::{AccessMode, KeyManagementSettings, KeyManagementSystem, SecretValue};
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::Value;
|
||||
|
||||
use super::callback::PythonSecretManager;
|
||||
use crate::{
|
||||
coercion::{Field, FieldSpec, ProjectionError},
|
||||
python_settings::{PythonSettings, Snapshot},
|
||||
};
|
||||
|
||||
const SYSTEM: FieldSpec<Option<KeyManagementSystem>> =
|
||||
FieldSpec::new("system", parse_optional_system);
|
||||
const ACCESS_MODE: FieldSpec<AccessMode> = FieldSpec::new("access_mode", parse_access_mode);
|
||||
const HOSTED_KEYS: FieldSpec<Option<Vec<String>>> =
|
||||
FieldSpec::new("hosted_keys", |field| field.optional_string_collection());
|
||||
const STORE_VIRTUAL_KEYS: FieldSpec<bool> =
|
||||
FieldSpec::new("store_virtual_keys", |field| field.truthy());
|
||||
const PREFIX_FOR_STORED_VIRTUAL_KEYS: FieldSpec<String> =
|
||||
FieldSpec::new("prefix_for_stored_virtual_keys", |field| {
|
||||
field.strict_string()
|
||||
});
|
||||
const PRIMARY_SECRET_NAME: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("primary_secret_name", |field| field.falsy_optional_string());
|
||||
const KMS_KEY_ID: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("kms_key_id", |field| field.falsy_optional_string());
|
||||
const CUSTOM_SECRET_MANAGER: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("custom_secret_manager", |field| {
|
||||
field.falsy_optional_string()
|
||||
});
|
||||
const AWS_REGION_NAME: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("aws_region_name", |field| field.falsy_optional_string());
|
||||
const AWS_ROLE_NAME: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("aws_role_name", |field| field.falsy_optional_string());
|
||||
const AWS_SESSION_NAME: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("aws_session_name", |field| field.falsy_optional_string());
|
||||
const AWS_EXTERNAL_ID: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("aws_external_id", |field| field.falsy_optional_string());
|
||||
const AWS_PROFILE_NAME: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("aws_profile_name", |field| field.falsy_optional_string());
|
||||
const AWS_WEB_IDENTITY_TOKEN: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("aws_web_identity_token", |field| {
|
||||
field.falsy_optional_string()
|
||||
});
|
||||
const AWS_STS_ENDPOINT: FieldSpec<Option<String>> =
|
||||
FieldSpec::new("aws_sts_endpoint", |field| field.falsy_optional_string());
|
||||
const REPLICA_REGIONS: FieldSpec<Option<Vec<String>>> =
|
||||
FieldSpec::new("replica_regions", |field| {
|
||||
field.optional_string_collection()
|
||||
});
|
||||
const CLIENT: FieldSpec<Option<Py<PyAny>>> =
|
||||
FieldSpec::new("client", |field| Ok(field.python_binding()));
|
||||
const SETTINGS_OBJECT: FieldSpec<Option<Py<PyAny>>> =
|
||||
FieldSpec::new("settings_object", |field| Ok(field.python_binding()));
|
||||
|
||||
/// `litellm.secret_manager_client` as the bridge classifies it.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum SecretManagerClient {
|
||||
/// `None`: reads come from the process environment.
|
||||
Local,
|
||||
/// A custom manager, legacy compatible client, or manually assigned SDK client that keeps
|
||||
/// executing in Python.
|
||||
PythonCallback(Py<PyAny>),
|
||||
}
|
||||
|
||||
/// One operation-local capture of the secret manager globals, taken while attached to Python.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct SecretManagerSnapshot {
|
||||
pub(crate) client: SecretManagerClient,
|
||||
pub(crate) system: Option<KeyManagementSystem>,
|
||||
/// Typed settings that drive native routing: access mode and hosted keys.
|
||||
pub(crate) settings: KeyManagementSettings,
|
||||
/// The original `KeyManagementSettings` object, handed back to Python callbacks unchanged.
|
||||
pub(crate) settings_object: Option<Py<PyAny>>,
|
||||
}
|
||||
|
||||
impl SecretManagerSnapshot {
|
||||
pub(crate) fn into_state(self) -> Arc<SecretManagerState> {
|
||||
match self.client {
|
||||
SecretManagerClient::Local => Arc::new(SecretManagerState::default()),
|
||||
SecretManagerClient::PythonCallback(client) => Arc::new(SecretManagerState::new(
|
||||
SecretManager::External(Arc::new(PythonSecretManager::new(
|
||||
client,
|
||||
self.system,
|
||||
self.settings_object,
|
||||
))),
|
||||
self.settings,
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads and projects the secret manager settings group in one attached operation.
|
||||
pub(crate) fn read(py: Python<'_>) -> PyResult<SecretManagerSnapshot> {
|
||||
Ok(project(&PythonSettings::SecretManagerBinding.read(py)?)?)
|
||||
}
|
||||
|
||||
pub(crate) fn project(snapshot: &Snapshot<'_>) -> Result<SecretManagerSnapshot, ProjectionError> {
|
||||
let system = snapshot.read(&SYSTEM)?;
|
||||
let access_mode = snapshot.read(&ACCESS_MODE)?;
|
||||
let settings = KeyManagementSettings {
|
||||
hosted_keys: snapshot.read(&HOSTED_KEYS)?,
|
||||
store_virtual_keys: Some(snapshot.read(&STORE_VIRTUAL_KEYS)?),
|
||||
prefix_for_stored_virtual_keys: snapshot.read(&PREFIX_FOR_STORED_VIRTUAL_KEYS)?,
|
||||
access_mode,
|
||||
primary_secret_name: snapshot.read(&PRIMARY_SECRET_NAME)?,
|
||||
kms_key_id: snapshot.read(&KMS_KEY_ID)?,
|
||||
custom_secret_manager: snapshot.read(&CUSTOM_SECRET_MANAGER)?,
|
||||
aws_region_name: snapshot.read(&AWS_REGION_NAME)?,
|
||||
aws_role_name: snapshot.read(&AWS_ROLE_NAME)?,
|
||||
aws_session_name: snapshot.read(&AWS_SESSION_NAME)?,
|
||||
aws_external_id: snapshot.read(&AWS_EXTERNAL_ID)?.map(SecretValue::new),
|
||||
aws_profile_name: snapshot.read(&AWS_PROFILE_NAME)?,
|
||||
aws_web_identity_token: snapshot
|
||||
.read(&AWS_WEB_IDENTITY_TOKEN)?
|
||||
.map(SecretValue::new),
|
||||
aws_sts_endpoint: snapshot.read(&AWS_STS_ENDPOINT)?,
|
||||
replica_regions: snapshot.read(&REPLICA_REGIONS)?,
|
||||
..KeyManagementSettings::default()
|
||||
};
|
||||
let client = match snapshot.read(&CLIENT)? {
|
||||
None => SecretManagerClient::Local,
|
||||
Some(client) => SecretManagerClient::PythonCallback(client),
|
||||
};
|
||||
Ok(SecretManagerSnapshot {
|
||||
client,
|
||||
system,
|
||||
settings,
|
||||
settings_object: snapshot.read(&SETTINGS_OBJECT)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_optional_system(
|
||||
field: &Field<'_>,
|
||||
) -> Result<Option<KeyManagementSystem>, ProjectionError> {
|
||||
let Some(value) = field.falsy_optional_string()? else {
|
||||
return Ok(None);
|
||||
};
|
||||
serde_json::from_value(Value::String(value))
|
||||
.map(Some)
|
||||
.map_err(|error| {
|
||||
ProjectionError::InvalidConfiguration(format!("secret manager system: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_access_mode(field: &Field<'_>) -> Result<AccessMode, ProjectionError> {
|
||||
let value = field.strict_string()?;
|
||||
serde_json::from_value(Value::String(value)).map_err(|error| {
|
||||
ProjectionError::InvalidConfiguration(format!("secret manager access mode: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use pyo3::{
|
||||
prelude::*,
|
||||
types::{PyDict, PyTuple},
|
||||
};
|
||||
|
||||
use super::{SecretManagerClient, project};
|
||||
use crate::python_settings::PythonSettings;
|
||||
|
||||
fn snapshot<'py>(
|
||||
py: Python<'py>,
|
||||
system: &str,
|
||||
access_mode: &str,
|
||||
store_virtual_keys: Bound<'py, PyAny>,
|
||||
hosted_keys: Bound<'py, PyAny>,
|
||||
) -> crate::python_settings::Snapshot<'py> {
|
||||
snapshot_with_client(
|
||||
py,
|
||||
system,
|
||||
access_mode,
|
||||
store_virtual_keys,
|
||||
hosted_keys,
|
||||
py.None().into_bound(py),
|
||||
)
|
||||
}
|
||||
|
||||
fn snapshot_with_client<'py>(
|
||||
py: Python<'py>,
|
||||
system: &str,
|
||||
access_mode: &str,
|
||||
store_virtual_keys: Bound<'py, PyAny>,
|
||||
hosted_keys: Bound<'py, PyAny>,
|
||||
client: Bound<'py, PyAny>,
|
||||
) -> crate::python_settings::Snapshot<'py> {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("client", client).unwrap();
|
||||
locals.set_item("system", system).unwrap();
|
||||
locals.set_item("access_mode", access_mode).unwrap();
|
||||
locals
|
||||
.set_item("store_virtual_keys", store_virtual_keys)
|
||||
.unwrap();
|
||||
locals.set_item("hosted_keys", hosted_keys).unwrap();
|
||||
py.run(
|
||||
cr#"
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SecretManager:
|
||||
system: object
|
||||
access_mode: object
|
||||
hosted_keys: object
|
||||
primary_secret_name: object
|
||||
store_virtual_keys: object
|
||||
prefix_for_stored_virtual_keys: object
|
||||
kms_key_id: object
|
||||
custom_secret_manager: object
|
||||
aws_region_name: object
|
||||
aws_role_name: object
|
||||
aws_session_name: object
|
||||
aws_external_id: object
|
||||
aws_profile_name: object
|
||||
aws_web_identity_token: object
|
||||
aws_sts_endpoint: object
|
||||
replica_regions: object
|
||||
client: object
|
||||
settings_object: object
|
||||
|
||||
root = SimpleNamespace(secret_manager=SecretManager(
|
||||
system=system,
|
||||
access_mode=access_mode,
|
||||
hosted_keys=hosted_keys,
|
||||
primary_secret_name=None,
|
||||
store_virtual_keys=store_virtual_keys,
|
||||
prefix_for_stored_virtual_keys="litellm/",
|
||||
kms_key_id=None,
|
||||
custom_secret_manager=None,
|
||||
aws_region_name=None,
|
||||
aws_role_name=None,
|
||||
aws_session_name=None,
|
||||
aws_external_id=None,
|
||||
aws_profile_name=None,
|
||||
aws_web_identity_token=None,
|
||||
aws_sts_endpoint=None,
|
||||
replica_regions=None,
|
||||
client=client,
|
||||
settings_object=None,
|
||||
))
|
||||
"#,
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
PythonSettings::SecretManagerBinding.snapshot(
|
||||
locals
|
||||
.get_item("root")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.getattr("secret_manager")
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::string_true(Some("true"), false, true)]
|
||||
#[case::string_one(Some("1"), false, true)]
|
||||
#[case::true_value(None, true, true)]
|
||||
#[case::false_value(None, false, false)]
|
||||
#[case::string_false(Some("false"), false, true)]
|
||||
fn python_compatible_boolean_coercion(
|
||||
#[case] string_value: Option<&str>,
|
||||
#[case] bool_value: bool,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let store_virtual_keys = match string_value {
|
||||
Some(value) => value.into_pyobject(py).unwrap().into_any(),
|
||||
None => bool_value.into_pyobject(py).unwrap().to_owned().into_any(),
|
||||
};
|
||||
let hosted_keys = PyTuple::new(py, ["ONE"]).unwrap().into_any();
|
||||
let projected = project(&snapshot(
|
||||
py,
|
||||
"local",
|
||||
"read_only",
|
||||
store_virtual_keys,
|
||||
hosted_keys,
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(projected.settings.store_virtual_keys, Some(expected));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_system_is_rejected() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let error = project(&snapshot(
|
||||
py,
|
||||
"unknown",
|
||||
"read_only",
|
||||
false.into_pyobject(py).unwrap().to_owned().into_any(),
|
||||
PyTuple::empty(py).into_any(),
|
||||
))
|
||||
.unwrap_err();
|
||||
let error: PyErr = error.into();
|
||||
assert!(error.is_instance_of::<pyo3::exceptions::PyValueError>(py));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_identity_selects_local_or_python_callback() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let falsy = false.into_pyobject(py).unwrap().to_owned().into_any();
|
||||
let local = project(&snapshot(
|
||||
py,
|
||||
"local",
|
||||
"read_only",
|
||||
falsy.clone(),
|
||||
PyTuple::empty(py).into_any(),
|
||||
))
|
||||
.unwrap();
|
||||
assert!(matches!(local.client, SecretManagerClient::Local));
|
||||
assert!(local.settings_object.is_none());
|
||||
|
||||
let manager = py.eval(c"object()", None, None).unwrap();
|
||||
let custom = project(&snapshot_with_client(
|
||||
py,
|
||||
"custom",
|
||||
"read_only",
|
||||
falsy,
|
||||
PyTuple::empty(py).into_any(),
|
||||
manager.clone(),
|
||||
))
|
||||
.unwrap();
|
||||
let SecretManagerClient::PythonCallback(client) = custom.client else {
|
||||
panic!("a live client must stay a Python callback");
|
||||
};
|
||||
assert!(client.bind(py).is(&manager));
|
||||
});
|
||||
}
|
||||
}
|
||||
3
litellm-rust/crates/python-bridge/src/secrets/mod.rs
Normal file
3
litellm-rust/crates/python-bridge/src/secrets/mod.rs
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
pub(crate) mod callback;
|
||||
pub(crate) mod config;
|
||||
pub(crate) mod resolved;
|
||||
252
litellm-rust/crates/python-bridge/src/secrets/resolved.rs
Normal file
252
litellm-rust/crates/python-bridge/src/secrets/resolved.rs
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
use futures_util::{future::BoxFuture, future::try_join_all};
|
||||
use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
|
||||
use litellm_llms::base_llm::inference::secrets::{SecretSource, Secrets};
|
||||
use litellm_secrets::{
|
||||
Error, FailurePolicy, OidcResolver, Secret, SecretManagerState, SecretResolver,
|
||||
};
|
||||
|
||||
use super::config::SecretManagerSnapshot;
|
||||
|
||||
pub(crate) struct ResolvedSecrets {
|
||||
resolver: SecretResolver,
|
||||
}
|
||||
|
||||
impl ResolvedSecrets {
|
||||
pub(crate) fn new(snapshot: SecretManagerSnapshot) -> Self {
|
||||
Self::from_state(snapshot.into_state())
|
||||
}
|
||||
|
||||
fn from_state(state: Arc<SecretManagerState>) -> Self {
|
||||
Self {
|
||||
resolver: SecretResolver::new(
|
||||
state,
|
||||
Arc::new(ProcessEnvironment),
|
||||
OidcResolver::default(),
|
||||
)
|
||||
.with_failure_policy(FailurePolicy::EnvironmentFallback),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SecretSource for ResolvedSecrets {
|
||||
fn resolve<'a>(&'a self, names: &'a [&'static str]) -> BoxFuture<'a, Result<Secrets, Error>> {
|
||||
Box::pin(async move {
|
||||
let values = try_join_all(names.iter().map(|name| async move {
|
||||
self.resolver
|
||||
.get_secret(name, None)
|
||||
.await
|
||||
.map(|secret| secret.map(|secret| ((*name).to_owned(), secret_value(secret))))
|
||||
}))
|
||||
.await?
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<HashMap<_, _>>();
|
||||
Ok(Arc::new(ResolvedLookup { values }) as Secrets)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct ResolvedLookup {
|
||||
values: HashMap<String, String>,
|
||||
}
|
||||
|
||||
impl Lookup for ResolvedLookup {
|
||||
fn get(&self, name: &str) -> Option<String> {
|
||||
self.values
|
||||
.get(name)
|
||||
.cloned()
|
||||
.or_else(|| ProcessEnvironment.get(name))
|
||||
}
|
||||
}
|
||||
|
||||
fn secret_value(secret: Secret) -> String {
|
||||
match secret {
|
||||
Secret::String(value) => value.expose().to_owned(),
|
||||
Secret::Bool(value) => if value { "True" } else { "False" }.to_owned(),
|
||||
Secret::Json(value) => value.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use aws_sdk_secretsmanager::Client;
|
||||
use aws_sdk_secretsmanager::config::{
|
||||
BehaviorVersion, Credentials, Region, retry::RetryConfig,
|
||||
};
|
||||
use litellm_secrets::{AccessMode, KeyManagementSettings, SecretManager, SecretManagerState};
|
||||
use litellm_secrets_aws::AwsSecretsManagerV2;
|
||||
use serde_json::json;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{body_partial_json, header},
|
||||
};
|
||||
|
||||
use super::ResolvedSecrets;
|
||||
use litellm_llms::base_llm::inference::secrets::SecretSource;
|
||||
|
||||
fn state(server: &MockServer, settings: KeyManagementSettings) -> Arc<SecretManagerState> {
|
||||
let client = Client::from_conf(
|
||||
aws_sdk_secretsmanager::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new("us-east-1"))
|
||||
.credentials_provider(Credentials::new("test", "test", None, None, "test"))
|
||||
.endpoint_url(server.uri())
|
||||
.retry_config(RetryConfig::disabled())
|
||||
.build(),
|
||||
);
|
||||
Arc::new(SecretManagerState::new(
|
||||
SecretManager::AwsSecretsManagerV2(AwsSecretsManagerV2::new(
|
||||
client,
|
||||
(&settings).into(),
|
||||
)),
|
||||
settings,
|
||||
))
|
||||
}
|
||||
|
||||
async fn resolve(state: Arc<SecretManagerState>, name: &'static str) -> Option<String> {
|
||||
ResolvedSecrets::from_state(state)
|
||||
.resolve(&[name])
|
||||
.await
|
||||
.unwrap()
|
||||
.get(name)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn hosted_key_miss_falls_back_to_environment() {
|
||||
let name = "LITELLM_RUST_BRIDGE_HOSTED_KEY_MISS";
|
||||
unsafe { std::env::set_var(name, "env-key") };
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.and(body_partial_json(json!({"SecretId": name})))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})),
|
||||
)
|
||||
.expect(0)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let result = resolve(
|
||||
state(
|
||||
&server,
|
||||
KeyManagementSettings {
|
||||
hosted_keys: Some(vec!["OTHER".into()]),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
name,
|
||||
)
|
||||
.await;
|
||||
unsafe { std::env::remove_var(name) };
|
||||
assert_eq!(result.as_deref(), Some("env-key"));
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manager_failure_falls_back_to_environment() {
|
||||
let name = "LITELLM_RUST_BRIDGE_MANAGER_FAILURE";
|
||||
unsafe { std::env::set_var(name, "env-key") };
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let result = resolve(state(&server, KeyManagementSettings::default()), name).await;
|
||||
unsafe { std::env::remove_var(name) };
|
||||
assert_eq!(result.as_deref(), Some("env-key"));
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
|
||||
let missing_server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.expect(1)
|
||||
.mount(&missing_server)
|
||||
.await;
|
||||
let missing =
|
||||
ResolvedSecrets::from_state(state(&missing_server, KeyManagementSettings::default()))
|
||||
.resolve(&["LITELLM_RUST_BRIDGE_MANAGER_FAILURE_MISSING"])
|
||||
.await;
|
||||
assert!(matches!(missing, Err(litellm_secrets::Error::Aws(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_only_mode_never_consults_the_manager() {
|
||||
let name = "LITELLM_RUST_BRIDGE_WRITE_ONLY";
|
||||
unsafe { std::env::set_var(name, "env-key") };
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})),
|
||||
)
|
||||
.expect(0)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let result = resolve(
|
||||
state(
|
||||
&server,
|
||||
KeyManagementSettings {
|
||||
access_mode: AccessMode::WriteOnly,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
name,
|
||||
)
|
||||
.await;
|
||||
unsafe { std::env::remove_var(name) };
|
||||
assert_eq!(result.as_deref(), Some("env-key"));
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_only_mode_resolves_from_the_manager() {
|
||||
let name = "LITELLM_RUST_BRIDGE_READ_ONLY";
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.and(body_partial_json(json!({"SecretId": name})))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "manager-key"})),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
assert_eq!(
|
||||
resolve(state(&server, KeyManagementSettings::default()), name)
|
||||
.await
|
||||
.as_deref(),
|
||||
Some("manager-key")
|
||||
);
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oidc_failures_are_not_converted_to_missing_secrets() {
|
||||
let result = ResolvedSecrets::from_state(Arc::new(SecretManagerState::default()))
|
||||
.resolve(&["oidc/"])
|
||||
.await;
|
||||
assert!(matches!(result, Err(litellm_secrets::Error::InvalidOidc)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn undeclared_names_still_read_the_process_environment() {
|
||||
let name = "LITELLM_RUST_BRIDGE_UNDECLARED";
|
||||
unsafe { std::env::set_var(name, "env-key") };
|
||||
let server = MockServer::start().await;
|
||||
let result = resolve(
|
||||
state(
|
||||
&server,
|
||||
KeyManagementSettings {
|
||||
hosted_keys: Some(vec!["OTHER".into()]),
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
name,
|
||||
)
|
||||
.await;
|
||||
unsafe { std::env::remove_var(name) };
|
||||
assert_eq!(result.as_deref(), Some("env-key"));
|
||||
assert_eq!(server.received_requests().await.unwrap().len(), 0);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_host_python::run_async;
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
types::PyAny,
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
/// Counts the input tokens of a raw request body off the Python event loop with
|
||||
/// the GIL released. Python owns which requests get here and what to do with
|
||||
/// the count. At most one encode per core runs at a time; the rest wait in the
|
||||
/// async task, where a cancelled Python awaiter drops them before any blocking
|
||||
/// work is scheduled.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct TokenCounter {
|
||||
inner: Arc<CoreTokenCounter>,
|
||||
encode_slots: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl TokenCounter {
|
||||
#[new]
|
||||
fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json))
|
||||
}
|
||||
#[cfg(all(not(feature = "fast"), feature = "huggingface"))]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
|
||||
}
|
||||
#[cfg(not(any(feature = "fast", feature = "huggingface")))]
|
||||
{
|
||||
let _ = (py, tokenizer_json);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast or huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
{
|
||||
let _ = (py, rank_file);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
{
|
||||
let _ = (py, rank_file);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_tiktoken(encoding))
|
||||
}
|
||||
#[cfg(not(feature = "tiktoken"))]
|
||||
{
|
||||
let _ = (py, encoding);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the tiktoken feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
|
||||
let counter = Arc::clone(&self.inner);
|
||||
let encode_slots = Arc::clone(&self.encode_slots);
|
||||
let body = body.to_vec();
|
||||
run_async(
|
||||
py,
|
||||
async move {
|
||||
let _slot = encode_slots
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?;
|
||||
tokio::task::spawn_blocking(move || count_body(&counter, &body))
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?
|
||||
},
|
||||
token_count_error_to_pyerr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenCounter {
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
fn load(
|
||||
py: Python<'_>,
|
||||
load: impl FnOnce() -> Result<CoreTokenCounter, Error> + Send,
|
||||
) -> PyResult<Self> {
|
||||
let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?;
|
||||
Ok(Self {
|
||||
inner: Arc::new(inner),
|
||||
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
fn encode_parallelism() -> usize {
|
||||
available_parallelism().map_or(1, NonZero::get)
|
||||
}
|
||||
|
||||
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
|
||||
let request = CountableRequest::parse(body)?;
|
||||
counter.count_request(&request)
|
||||
}
|
||||
|
||||
fn token_count_error_to_pyerr(error: Error) -> PyErr {
|
||||
let message = error.to_string();
|
||||
match error {
|
||||
Error::Load(_)
|
||||
| Error::Ranks(_)
|
||||
| Error::UnicodeClasses
|
||||
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
|
||||
Error::RequestParse(_)
|
||||
| Error::MissingInput
|
||||
| Error::FloatText
|
||||
| Error::ContentBlock
|
||||
| Error::ArrayItems
|
||||
| Error::JsonSerialization(_)
|
||||
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
|
||||
Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
|
||||
}
|
||||
}
|
||||
713
litellm-rust/crates/python-bridge/src/tokenizer.rs
Normal file
713
litellm-rust/crates/python-bridge/src/tokenizer.rs
Normal file
|
|
@ -0,0 +1,713 @@
|
|||
//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging
|
||||
//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and
|
||||
//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps.
|
||||
use std::borrow::Cow;
|
||||
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
#[cfg(feature = "fast")]
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use litellm_host_python::{enter_native, release_gil};
|
||||
#[cfg(feature = "fast")]
|
||||
use litellm_token_counter::fast::{FastCounter, FastTokenizer};
|
||||
use litellm_token_counter::{Error, TextCodec};
|
||||
use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString};
|
||||
|
||||
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
|
||||
use pyo3::exceptions::PyValueError;
|
||||
#[cfg(feature = "huggingface")]
|
||||
use pyo3::{exceptions::PyIOError, types::PyDict};
|
||||
#[cfg(feature = "tiktoken")]
|
||||
use pyo3::{
|
||||
exceptions::{PyKeyError, PyRuntimeError},
|
||||
types::PyBytes,
|
||||
};
|
||||
|
||||
#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))]
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::routes::token_counter::token_count_error_to_pyerr;
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
use litellm_token_counter::huggingface::{
|
||||
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy,
|
||||
TruncationDirection, encoding_from_json, encoding_to_json,
|
||||
};
|
||||
#[cfg(feature = "tiktoken")]
|
||||
use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary};
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<TiktokenTokenizer> {
|
||||
enter_native()?;
|
||||
let resource: std::path::PathBuf =
|
||||
PyModule::import(py, "litellm.litellm_core_utils.tokenizers")?
|
||||
.getattr("__file__")?
|
||||
.extract()?;
|
||||
release_gil(py, || {
|
||||
TiktokenTokenizer::from_cached_ranks(encoding, |file| {
|
||||
std::fs::read_to_string(resource.with_file_name(file))
|
||||
})
|
||||
})
|
||||
.map_err(|error| token_count_error_to_pyerr(error.into()))
|
||||
}
|
||||
|
||||
pub(crate) enum Codec {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Tiktoken(TiktokenTokenizer),
|
||||
#[cfg(feature = "huggingface")]
|
||||
HuggingFace(HuggingFaceTokenizer),
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub(crate) fn codec(&self) -> &dyn TextCodec {
|
||||
match *self {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Self::Tiktoken(ref tokenizer) => tokenizer,
|
||||
#[cfg(feature = "huggingface")]
|
||||
Self::HuggingFace(ref tokenizer) => tokenizer,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fast")]
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
match *self {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(),
|
||||
#[cfg(feature = "huggingface")]
|
||||
Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse,
|
||||
/// and the opt-in count-only counter is derived from it once, on first use.
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Tokenizer {
|
||||
inner: Arc<Codec>,
|
||||
#[cfg(feature = "fast")]
|
||||
fast: OnceLock<Option<Arc<FastTokenizer>>>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Tokenizer {
|
||||
#[staticmethod]
|
||||
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
{
|
||||
let tokenizer = load_tiktoken(py, encoding)?;
|
||||
Ok(Self::new(Codec::Tiktoken(tokenizer)))
|
||||
}
|
||||
#[cfg(not(feature = "tiktoken"))]
|
||||
{
|
||||
let _ = (py, encoding);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the tiktoken feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "huggingface")]
|
||||
{
|
||||
enter_native()?;
|
||||
let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json))
|
||||
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
|
||||
Ok(Self::new(Codec::HuggingFace(tokenizer)))
|
||||
}
|
||||
#[cfg(not(feature = "huggingface"))]
|
||||
{
|
||||
let _ = (py, tokenizer_json);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (identifier, revision = "main", token = None))]
|
||||
fn from_pretrained(
|
||||
py: Python<'_>,
|
||||
identifier: &str,
|
||||
revision: &str,
|
||||
token: Option<&str>,
|
||||
) -> PyResult<Self> {
|
||||
#[cfg(feature = "huggingface")]
|
||||
{
|
||||
enter_native()?;
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("repo_id", identifier)?;
|
||||
kwargs.set_item("filename", "tokenizer.json")?;
|
||||
kwargs.set_item("revision", revision)?;
|
||||
kwargs.set_item("token", token)?;
|
||||
let path: String = PyModule::import(py, "huggingface_hub")?
|
||||
.getattr("hf_hub_download")?
|
||||
.call((), Some(&kwargs))?
|
||||
.extract()?;
|
||||
let json =
|
||||
release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?;
|
||||
Self::from_json(py, &json)
|
||||
}
|
||||
#[cfg(not(feature = "huggingface"))]
|
||||
{
|
||||
let _ = (py, identifier, revision, token);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<Vec<u32>> {
|
||||
enter_native()?;
|
||||
let text = self.text(text)?;
|
||||
release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (ids, skip_special_tokens = true))]
|
||||
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
|
||||
enter_native()?;
|
||||
release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens))
|
||||
.map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (text, fast = false))]
|
||||
fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult<usize> {
|
||||
enter_native()?;
|
||||
let text = self.text(text)?;
|
||||
let counter = self.counter(py, fast);
|
||||
release_gil(py, || {
|
||||
litellm_token_counter::Tokenizer::count_tokens(&counter, &text)
|
||||
})
|
||||
.map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn name(&self) -> &str {
|
||||
self.inner.codec().name()
|
||||
}
|
||||
|
||||
// ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_special(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
text: &Bound<'_, PyString>,
|
||||
allowed: Vec<String>,
|
||||
) -> PyResult<Vec<u32>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let text = self.text(text)?;
|
||||
release_gil(py, || tokenizer.encode_special(&text, &allowed))
|
||||
.map_err(PyRuntimeError::new_err)
|
||||
}
|
||||
|
||||
/// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_with_unstable(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
text: &Bound<'_, PyString>,
|
||||
allowed: Vec<String>,
|
||||
) -> PyResult<(Vec<u32>, Vec<Vec<u32>>)> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let text = self.text(text)?;
|
||||
Ok(release_gil(py, || {
|
||||
tokenizer.encode_with_unstable(&text, &allowed)
|
||||
}))
|
||||
}
|
||||
|
||||
/// The special tokens by text: tiktoken's `_special_tokens`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn special_tokens(&self) -> PyResult<HashMap<String, u32>> {
|
||||
Ok(self
|
||||
.vocabulary()?
|
||||
.special_tokens()
|
||||
.map(|(token, rank)| (token.to_owned(), rank))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn max_token_value(&self) -> PyResult<u32> {
|
||||
Ok(self.vocabulary()?.max_token_value())
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn is_special_token(&self, token: u32) -> PyResult<bool> {
|
||||
Ok(self.vocabulary()?.is_special_token(token))
|
||||
}
|
||||
|
||||
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyBytes>>> {
|
||||
let vocabulary = self.vocabulary()?;
|
||||
let values = release_gil(py, || vocabulary.token_byte_values());
|
||||
Ok(values.iter().map(|value| PyBytes::new(py, value)).collect())
|
||||
}
|
||||
|
||||
/// The token of one whole piece; `KeyError` when it is not in the vocabulary.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_single_token(&self, py: Python<'_>, piece: Vec<u8>) -> PyResult<u32> {
|
||||
self.vocabulary()?
|
||||
.encode_single_token(&piece)
|
||||
.ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind()))
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec<u32>) -> PyResult<Bound<'py, PyBytes>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let bytes =
|
||||
release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?;
|
||||
Ok(PyBytes::new(py, &bytes))
|
||||
}
|
||||
|
||||
// ---- Hugging Face: the `tokenizers.Tokenizer` surface -----------------------------------
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))]
|
||||
fn encode_huggingface(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
sequence: Sequence,
|
||||
pair: Option<Sequence>,
|
||||
is_pretokenized: bool,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> PyResult<HuggingFaceEncoding> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
let sequence = sequence.input(is_pretokenized)?;
|
||||
let input = match pair {
|
||||
Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?),
|
||||
None => EncodeInput::Single(sequence),
|
||||
};
|
||||
release_gil(py, || {
|
||||
tokenizer.encode_result(input, add_special_tokens, fast)
|
||||
})
|
||||
.map(|inner| HuggingFaceEncoding { inner })
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))]
|
||||
fn encode_batch_huggingface(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
inputs: Vec<(Sequence, Option<Sequence>)>,
|
||||
is_pretokenized: bool,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> PyResult<Vec<HuggingFaceEncoding>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
let inputs = inputs
|
||||
.into_iter()
|
||||
.map(|(sequence, pair)| {
|
||||
let sequence = sequence.input(is_pretokenized)?;
|
||||
match pair {
|
||||
Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)),
|
||||
None => Ok(EncodeInput::Single(sequence)),
|
||||
}
|
||||
})
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
release_gil(py, || {
|
||||
tokenizer.encode_batch_result(inputs, add_special_tokens, fast)
|
||||
})
|
||||
.map(|encodings| {
|
||||
encodings
|
||||
.into_iter()
|
||||
.map(|inner| HuggingFaceEncoding { inner })
|
||||
.collect()
|
||||
})
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (pretty = false))]
|
||||
fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult<String> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
release_gil(py, || tokenizer.to_json(pretty))
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn token_to_id(&self, token: &str) -> PyResult<Option<u32>> {
|
||||
Ok(self.huggingface()?.token_to_id(token))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn id_to_token(&self, id: u32) -> PyResult<Option<String>> {
|
||||
Ok(self.huggingface()?.id_to_token(id))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (with_added_tokens = true))]
|
||||
fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult<HashMap<String, u32>> {
|
||||
let tokenizer = self.huggingface()?;
|
||||
Ok(release_gil(py, || tokenizer.vocab(with_added_tokens)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (with_added_tokens = true))]
|
||||
fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult<usize> {
|
||||
Ok(self.huggingface()?.vocab_size(with_added_tokens))
|
||||
}
|
||||
|
||||
/// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized,
|
||||
/// special))`, for Python to rebuild as `tokenizers.AddedToken`.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn added_tokens_decoder(&self) -> PyResult<Vec<(u32, AddedTokenFields)>> {
|
||||
Ok(self
|
||||
.huggingface()?
|
||||
.added_tokens_decoder()
|
||||
.into_iter()
|
||||
.map(|(id, token)| {
|
||||
(
|
||||
id,
|
||||
(
|
||||
token.content,
|
||||
token.single_word,
|
||||
token.lstrip,
|
||||
token.rstrip,
|
||||
token.normalized,
|
||||
token.special,
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// The padding parameters as `tokenizers.Tokenizer.padding` reports them.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn padding<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let Some(params) = self.huggingface()?.padding() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let padding = PyDict::new(py);
|
||||
padding.set_item(
|
||||
"length",
|
||||
match params.strategy {
|
||||
PaddingStrategy::BatchLongest => None,
|
||||
PaddingStrategy::Fixed(length) => Some(length),
|
||||
},
|
||||
)?;
|
||||
padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?;
|
||||
padding.set_item("pad_id", params.pad_id)?;
|
||||
padding.set_item("pad_type_id", params.pad_type_id)?;
|
||||
padding.set_item("pad_token", ¶ms.pad_token)?;
|
||||
padding.set_item("direction", params.direction.as_ref())?;
|
||||
Ok(Some(padding))
|
||||
}
|
||||
|
||||
/// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn truncation<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let Some(params) = self.huggingface()?.truncation() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let truncation = PyDict::new(py);
|
||||
truncation.set_item("max_length", params.max_length)?;
|
||||
truncation.set_item("stride", params.stride)?;
|
||||
truncation.set_item("strategy", params.strategy.as_ref())?;
|
||||
truncation.set_item("direction", params.direction.as_ref())?;
|
||||
Ok(Some(truncation))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult<usize> {
|
||||
Ok(self.huggingface()?.num_special_tokens_to_add(is_pair))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn encode_special_tokens(&self) -> PyResult<bool> {
|
||||
Ok(self.huggingface()?.encode_special_tokens())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
type AddedTokenFields = (String, bool, bool, bool, bool, bool);
|
||||
|
||||
impl Tokenizer {
|
||||
fn new(inner: Codec) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(inner),
|
||||
#[cfg(feature = "fast")]
|
||||
fast: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter {
|
||||
#[cfg(feature = "fast")]
|
||||
if fast {
|
||||
let counter = self.fast.get().unwrap_or_else(|| {
|
||||
release_gil(py, || {
|
||||
self.fast
|
||||
.get_or_init(|| self.inner.fast_counter().map(Arc::new))
|
||||
})
|
||||
});
|
||||
if let Some(counter) = counter {
|
||||
return SharedCounter::Fast(Arc::clone(counter));
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
let _ = (py, fast);
|
||||
SharedCounter::Codec(Arc::clone(&self.inner))
|
||||
}
|
||||
|
||||
/// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode`
|
||||
/// does; `tokenizers` rejects them, so that backend keeps the encode error.
|
||||
fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult<Cow<'a, str>> {
|
||||
match text.to_cow() {
|
||||
Ok(text) => Ok(text),
|
||||
Err(error) => match *self.inner {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Codec::Tiktoken(_) if error.is_instance_of::<PyUnicodeEncodeError>(text.py()) => {
|
||||
text.call_method1("encode", ("utf-16", "surrogatepass"))?
|
||||
.call_method1("decode", ("utf-16", "replace"))?
|
||||
.extract::<String>()
|
||||
.map(Cow::Owned)
|
||||
}
|
||||
_ => Err(error),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> {
|
||||
match *self.inner {
|
||||
Codec::Tiktoken(ref tokenizer) => Ok(tokenizer),
|
||||
#[cfg(feature = "huggingface")]
|
||||
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn vocabulary(&self) -> PyResult<&Vocabulary> {
|
||||
self.tiktoken()?.vocabulary().ok_or_else(|| {
|
||||
PyRuntimeError::new_err("this encoding was built without its vocabulary")
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> {
|
||||
match *self.inner {
|
||||
Codec::HuggingFace(ref tokenizer) => Ok(tokenizer),
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) enum SharedCounter {
|
||||
Codec(Arc<Codec>),
|
||||
#[cfg(feature = "fast")]
|
||||
Fast(Arc<FastTokenizer>),
|
||||
}
|
||||
|
||||
impl litellm_token_counter::Tokenizer for SharedCounter {
|
||||
fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
match self {
|
||||
Self::Codec(codec) => codec.codec().count_tokens(text),
|
||||
#[cfg(feature = "fast")]
|
||||
Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[derive(FromPyObject)]
|
||||
pub(crate) enum Sequence {
|
||||
Text(String),
|
||||
Words(Vec<String>),
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
impl Sequence {
|
||||
fn input(self, is_pretokenized: bool) -> PyResult<InputSequence<'static>> {
|
||||
match (self, is_pretokenized) {
|
||||
(Self::Text(text), false) => Ok(text.into()),
|
||||
(Self::Words(words), true) => Ok(words.into()),
|
||||
_ => Err(pyo3::exceptions::PyTypeError::new_err(
|
||||
"input must match is_pretokenized",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn direction<T>(value: &str, left: T, right: T, what: &str) -> PyResult<T> {
|
||||
match value {
|
||||
"left" => Ok(left),
|
||||
"right" => Ok(right),
|
||||
other => Err(PyValueError::new_err(format!(
|
||||
"invalid {what} direction {other:?}: expected 'left' or 'right'"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id`
|
||||
/// change it in place.
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyclass(module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct HuggingFaceEncoding {
|
||||
inner: Encoding,
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pymethods]
|
||||
impl HuggingFaceEncoding {
|
||||
#[new]
|
||||
#[pyo3(signature = (json = None))]
|
||||
fn new(json: Option<&str>) -> PyResult<Self> {
|
||||
let inner = match json {
|
||||
Some(json) => encoding_from_json(json)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?,
|
||||
None => Encoding::default(),
|
||||
};
|
||||
Ok(Self { inner })
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (encodings, growing_offsets = true))]
|
||||
fn merge(encodings: Vec<PyRef<'_, Self>>, growing_offsets: bool) -> Self {
|
||||
Self {
|
||||
inner: Encoding::merge(
|
||||
encodings.iter().map(|encoding| encoding.inner.clone()),
|
||||
growing_offsets,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn __reduce__<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
|
||||
let json = encoding_to_json(&self.inner)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
Ok((py.get_type::<Self>(), (json,)))
|
||||
}
|
||||
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \
|
||||
attention_mask, special_tokens_mask, overflowing])",
|
||||
self.inner.len()
|
||||
)
|
||||
}
|
||||
|
||||
fn __len__(&self) -> usize {
|
||||
self.inner.len()
|
||||
}
|
||||
#[getter]
|
||||
fn ids(&self) -> Vec<u32> {
|
||||
self.inner.get_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn tokens(&self) -> Vec<String> {
|
||||
self.inner.get_tokens().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn offsets(&self) -> Vec<(usize, usize)> {
|
||||
self.inner.get_offsets().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn type_ids(&self) -> Vec<u32> {
|
||||
self.inner.get_type_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn attention_mask(&self) -> Vec<u32> {
|
||||
self.inner.get_attention_mask().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn special_tokens_mask(&self) -> Vec<u32> {
|
||||
self.inner.get_special_tokens_mask().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn word_ids(&self) -> Vec<Option<u32>> {
|
||||
self.inner.get_word_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn sequence_ids(&self) -> Vec<Option<usize>> {
|
||||
self.inner.get_sequence_ids()
|
||||
}
|
||||
#[getter]
|
||||
fn overflowing(&self) -> Vec<Self> {
|
||||
self.inner
|
||||
.get_overflowing()
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|inner| Self { inner })
|
||||
.collect()
|
||||
}
|
||||
#[getter]
|
||||
fn n_sequences(&self) -> usize {
|
||||
self.inner.n_sequences()
|
||||
}
|
||||
|
||||
#[pyo3(signature = (word_index, sequence_index = 0))]
|
||||
fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner.word_to_tokens(word_index, sequence_index)
|
||||
}
|
||||
#[pyo3(signature = (word_index, sequence_index = 0))]
|
||||
fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner.word_to_chars(word_index, sequence_index)
|
||||
}
|
||||
fn token_to_sequence(&self, token_index: usize) -> Option<usize> {
|
||||
self.inner.token_to_sequence(token_index)
|
||||
}
|
||||
fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner
|
||||
.token_to_chars(token_index)
|
||||
.map(|(_, offsets)| offsets)
|
||||
}
|
||||
fn token_to_word(&self, token_index: usize) -> Option<u32> {
|
||||
self.inner.token_to_word(token_index).map(|(_, word)| word)
|
||||
}
|
||||
#[pyo3(signature = (char_pos, sequence_index = 0))]
|
||||
fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option<usize> {
|
||||
self.inner.char_to_token(char_pos, sequence_index)
|
||||
}
|
||||
#[pyo3(signature = (char_pos, sequence_index = 0))]
|
||||
fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option<u32> {
|
||||
self.inner.char_to_word(char_pos, sequence_index)
|
||||
}
|
||||
|
||||
fn set_sequence_id(&mut self, sequence_id: usize) {
|
||||
self.inner.set_sequence_id(sequence_id);
|
||||
}
|
||||
|
||||
#[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))]
|
||||
fn pad(
|
||||
&mut self,
|
||||
length: usize,
|
||||
direction: &str,
|
||||
pad_id: u32,
|
||||
pad_type_id: u32,
|
||||
pad_token: &str,
|
||||
) -> PyResult<()> {
|
||||
let direction = self::direction(
|
||||
direction,
|
||||
PaddingDirection::Left,
|
||||
PaddingDirection::Right,
|
||||
"padding",
|
||||
)?;
|
||||
self.inner
|
||||
.pad(length, pad_id, pad_type_id, pad_token, direction);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[pyo3(signature = (max_length, stride = 0, direction = "right"))]
|
||||
fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> {
|
||||
let direction = self::direction(
|
||||
direction,
|
||||
TruncationDirection::Left,
|
||||
TruncationDirection::Right,
|
||||
"truncation",
|
||||
)?;
|
||||
self.inner.truncate(max_length, stride, direction);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
|
|||
|
||||
use crate::SecretValue;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum KeyManagementSystem {
|
||||
GoogleKms,
|
||||
|
|
@ -18,7 +18,7 @@ pub enum KeyManagementSystem {
|
|||
Custom,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
|
||||
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum AccessMode {
|
||||
#[default]
|
||||
|
|
@ -33,7 +33,7 @@ impl AccessMode {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq, Hash)]
|
||||
#[serde(default)]
|
||||
pub struct KeyManagementSettings {
|
||||
pub hosted_keys: Option<Vec<String>>,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ pub enum Error {
|
|||
OidcFile,
|
||||
#[error("secret cannot be converted to {expected}")]
|
||||
TypeMismatch { expected: &'static str },
|
||||
#[error("external secret manager failed")]
|
||||
ExternalManager(#[source] Box<dyn std::error::Error + Send + Sync>),
|
||||
#[cfg(feature = "aws")]
|
||||
#[error(transparent)]
|
||||
Aws(#[from] litellm_secrets_aws::Error),
|
||||
|
|
|
|||
|
|
@ -1,10 +1,24 @@
|
|||
use std::{future::Future, pin::Pin, sync::Arc};
|
||||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
|
||||
use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue};
|
||||
|
||||
pub trait ExternalSecretManager: Send + Sync {
|
||||
fn system(&self) -> KeyManagementSystem;
|
||||
|
||||
fn read_secret<'a>(
|
||||
&'a self,
|
||||
name: &'a str,
|
||||
settings: &'a KeyManagementSettings,
|
||||
environment: &'a (dyn Lookup + Send + Sync),
|
||||
) -> Pin<Box<dyn Future<Output = Result<Option<Secret>, Error>> + Send + 'a>>;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum SecretManager {
|
||||
Local,
|
||||
External(Arc<dyn ExternalSecretManager>),
|
||||
#[cfg(feature = "aws")]
|
||||
AwsKms(crate::aws::AwsKms),
|
||||
#[cfg(feature = "aws")]
|
||||
|
|
@ -25,6 +39,7 @@ impl SecretManager {
|
|||
pub fn system(&self) -> KeyManagementSystem {
|
||||
match self {
|
||||
Self::Local => KeyManagementSystem::Local,
|
||||
Self::External(manager) => manager.system(),
|
||||
#[cfg(feature = "aws")]
|
||||
Self::AwsKms(_) => KeyManagementSystem::AwsKms,
|
||||
#[cfg(feature = "aws")]
|
||||
|
|
@ -54,6 +69,11 @@ pub async fn get_secret_from_manager(
|
|||
.get(secret_name)
|
||||
.map(SecretValue::new)
|
||||
.map(Secret::String)),
|
||||
SecretManager::External(manager) => {
|
||||
manager
|
||||
.read_secret(secret_name, _settings, environment)
|
||||
.await
|
||||
}
|
||||
#[cfg(feature = "aws")]
|
||||
SecretManager::AwsKms(client) => {
|
||||
let ciphertext = environment
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ mod resolver;
|
|||
mod state;
|
||||
|
||||
pub use error::Error;
|
||||
pub use handler::{SecretManager, get_secret_from_manager};
|
||||
pub use handler::{ExternalSecretManager, SecretManager, get_secret_from_manager};
|
||||
pub use litellm_secrets_types::{
|
||||
AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ impl SecretResolver {
|
|||
Ok(value) => Ok(value
|
||||
.or_else(|| self.environment_secret(name))
|
||||
.or(default_value)),
|
||||
Err(error @ Error::ExternalManager(_)) => Err(error),
|
||||
Err(error) => match self.failure_policy {
|
||||
FailurePolicy::Propagate => Err(error),
|
||||
FailurePolicy::EnvironmentFallback => self
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ mod scanner;
|
|||
mod tiktoken;
|
||||
mod unicode_classes;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use byte_level::ByteLevelCounter;
|
||||
use scanner::{SplitPattern, TiktokenCounter};
|
||||
|
||||
|
|
@ -15,22 +17,29 @@ pub use error::Error;
|
|||
|
||||
enum Encoder {
|
||||
HuggingFace {
|
||||
tokenizer: Box<tokenizers::Tokenizer>,
|
||||
tokenizer: Arc<tokenizers::Tokenizer>,
|
||||
byte_level: Option<ByteLevelCounter>,
|
||||
},
|
||||
Tiktoken(TiktokenCounter),
|
||||
}
|
||||
|
||||
/// A count-only tokenizer. Its model tables are immutable, so one built from an already
|
||||
/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables.
|
||||
pub struct FastTokenizer(Encoder);
|
||||
|
||||
impl FastTokenizer {
|
||||
pub fn from_json(json: &str) -> Result<Self, Error> {
|
||||
let tokenizer = json.parse::<tokenizers::Tokenizer>().map_err(Error::Load)?;
|
||||
Ok(Self::from_shared(Arc::new(tokenizer)))
|
||||
}
|
||||
|
||||
/// Counts with a Hugging Face model another codec already holds; nothing is re-parsed.
|
||||
pub fn from_shared(tokenizer: Arc<tokenizers::Tokenizer>) -> Self {
|
||||
let byte_level = ByteLevelCounter::detect(&tokenizer);
|
||||
Ok(Self(Encoder::HuggingFace {
|
||||
tokenizer: Box::new(tokenizer),
|
||||
Self(Encoder::HuggingFace {
|
||||
tokenizer,
|
||||
byte_level,
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn from_cl100k_ranks(ranks: &str) -> Result<Self, Error> {
|
||||
|
|
@ -41,12 +50,36 @@ impl FastTokenizer {
|
|||
Self::from_ranks(SplitPattern::O200k, ranks)
|
||||
}
|
||||
|
||||
/// `cl100k_base` from ranks another loader already parsed.
|
||||
pub fn from_cl100k_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_pairs(SplitPattern::Cl100k, pairs)
|
||||
}
|
||||
|
||||
/// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks
|
||||
/// another loader already parsed.
|
||||
pub fn from_o200k_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_pairs(SplitPattern::O200k, pairs)
|
||||
}
|
||||
|
||||
fn from_ranks(split: SplitPattern, ranks: &str) -> Result<Self, Error> {
|
||||
TiktokenCounter::from_ranks(split, ranks)
|
||||
.map(Encoder::Tiktoken)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
fn from_pairs<'a>(
|
||||
split: SplitPattern,
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
TiktokenCounter::from_pairs(split, pairs)
|
||||
.map(Encoder::Tiktoken)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
match &self.0 {
|
||||
Encoder::Tiktoken(counter) => Ok(counter.count(text)),
|
||||
|
|
|
|||
|
|
@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter {
|
|||
|
||||
impl TiktokenCounter {
|
||||
pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result<Self, Error> {
|
||||
Self::new(split, MergeRanks::parse(rank_file)?)
|
||||
}
|
||||
|
||||
pub(super) fn from_pairs<'a>(
|
||||
split: SplitPattern,
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::new(split, MergeRanks::from_pairs(pairs)?)
|
||||
}
|
||||
|
||||
fn new(split: SplitPattern, ranks: MergeRanks) -> Result<Self, Error> {
|
||||
Ok(Self {
|
||||
ranks: MergeRanks::parse(rank_file)?,
|
||||
ranks,
|
||||
piece_len: split.piece_len(),
|
||||
unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
|
|||
|
||||
impl MergeRanks {
|
||||
pub(super) fn parse(text: &str) -> Result<Self, Error> {
|
||||
let ranks = text
|
||||
.lines()
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(parse_line)
|
||||
.collect::<Result<FxHashMap<_, _>, _>>()?;
|
||||
Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line))
|
||||
}
|
||||
|
||||
/// The same table from ranks another loader already parsed.
|
||||
pub(super) fn from_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], Rank)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_entries(pairs.into_iter().map(|(bytes, rank)| {
|
||||
if rank == NO_RANK {
|
||||
return Err(Error::Ranks(format!("rank {rank} is reserved")));
|
||||
}
|
||||
Ok((Box::from(bytes), rank))
|
||||
}))
|
||||
}
|
||||
|
||||
fn from_entries(
|
||||
entries: impl Iterator<Item = Result<(Box<[u8]>, Rank), Error>>,
|
||||
) -> Result<Self, Error> {
|
||||
let ranks = entries.collect::<Result<FxHashMap<_, _>, _>>()?;
|
||||
if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
|
||||
return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,5 +6,6 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokenizers.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,4 +6,6 @@ pub enum Error {
|
|||
Load(#[source] tokenizers::Error),
|
||||
#[error("tokenization failed: {0}")]
|
||||
Encode(#[source] tokenizers::Error),
|
||||
#[error("token decoding failed: {0}")]
|
||||
Decode(#[source] tokenizers::Error),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,22 +2,243 @@
|
|||
|
||||
mod error;
|
||||
|
||||
pub use error::Error;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
|
||||
pub use error::Error;
|
||||
use tokenizers::PostProcessor;
|
||||
pub use tokenizers::{
|
||||
AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams,
|
||||
PaddingStrategy, TruncationDirection, TruncationParams,
|
||||
};
|
||||
|
||||
pub fn encoding_from_json(json: &str) -> Result<Encoding, Error> {
|
||||
serde_json::from_str(json).map_err(|error| Error::Load(error.into()))
|
||||
}
|
||||
|
||||
pub fn encoding_to_json(encoding: &Encoding) -> Result<String, Error> {
|
||||
serde_json::to_string(encoding).map_err(|error| Error::Load(error.into()))
|
||||
}
|
||||
|
||||
pub struct HuggingFaceTokenizer {
|
||||
tokenizer: Arc<tokenizers::Tokenizer>,
|
||||
special_token_ids: HashSet<u32>,
|
||||
}
|
||||
|
||||
impl HuggingFaceTokenizer {
|
||||
pub fn from_json(json: &str) -> Result<Self, Error> {
|
||||
json.parse::<tokenizers::Tokenizer>()
|
||||
.map(Box::new)
|
||||
.map(Self)
|
||||
.map(Self::new)
|
||||
.map_err(Error::Load)
|
||||
}
|
||||
|
||||
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
|
||||
let special_token_ids: HashSet<u32> = tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.into_iter()
|
||||
.filter_map(|(id, token)| token.special.then_some(id))
|
||||
.collect();
|
||||
Self {
|
||||
tokenizer: Arc::new(tokenizer),
|
||||
special_token_ids,
|
||||
}
|
||||
}
|
||||
|
||||
/// The parsed model, for a count-only counter to share instead of parsing it again.
|
||||
pub fn shared(&self) -> Arc<tokenizers::Tokenizer> {
|
||||
Arc::clone(&self.tokenizer)
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
self.0
|
||||
self.tokenizer
|
||||
.encode_fast(text, true)
|
||||
.map(|encoding| encoding.len())
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
self.tokenizer
|
||||
.encode_fast(text, true)
|
||||
.map(|encoding| encoding.get_ids().to_vec())
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode_result<'a>(
|
||||
&self,
|
||||
input: EncodeInput<'a>,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> Result<Encoding, Error> {
|
||||
if fast {
|
||||
return self
|
||||
.tokenizer
|
||||
.encode_fast(input, add_special_tokens)
|
||||
.map_err(Error::Encode);
|
||||
}
|
||||
self.tokenizer
|
||||
.encode_char_offsets(input, add_special_tokens)
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode_batch_result<'a>(
|
||||
&self,
|
||||
inputs: Vec<EncodeInput<'a>>,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> Result<Vec<Encoding>, Error> {
|
||||
if fast {
|
||||
return self
|
||||
.tokenizer
|
||||
.encode_batch_fast(inputs, add_special_tokens)
|
||||
.map_err(Error::Encode);
|
||||
}
|
||||
self.tokenizer
|
||||
.encode_batch_char_offsets(inputs, add_special_tokens)
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn to_json(&self, pretty: bool) -> Result<String, Error> {
|
||||
self.tokenizer.to_string(pretty).map_err(Error::Load)
|
||||
}
|
||||
|
||||
pub fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
self.tokenizer.token_to_id(token)
|
||||
}
|
||||
|
||||
pub fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
self.tokenizer.id_to_token(id)
|
||||
}
|
||||
|
||||
pub fn vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
|
||||
self.tokenizer.get_vocab(with_added_tokens)
|
||||
}
|
||||
|
||||
pub fn vocab_size(&self, with_added_tokens: bool) -> usize {
|
||||
self.tokenizer.get_vocab_size(with_added_tokens)
|
||||
}
|
||||
|
||||
/// The added tokens by id, in id order.
|
||||
pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> {
|
||||
let mut added: Vec<(u32, AddedToken)> = self
|
||||
.tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.into_iter()
|
||||
.collect();
|
||||
added.sort_unstable_by_key(|(id, _)| *id);
|
||||
added
|
||||
}
|
||||
|
||||
pub fn padding(&self) -> Option<&PaddingParams> {
|
||||
self.tokenizer.get_padding()
|
||||
}
|
||||
|
||||
pub fn truncation(&self) -> Option<&TruncationParams> {
|
||||
self.tokenizer.get_truncation()
|
||||
}
|
||||
|
||||
/// How many special tokens the post-processor adds to a single sequence or a pair.
|
||||
pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize {
|
||||
self.tokenizer
|
||||
.get_post_processor()
|
||||
.map_or(0, |processor| processor.added_tokens(is_pair))
|
||||
}
|
||||
|
||||
pub fn encode_special_tokens(&self) -> bool {
|
||||
self.tokenizer.get_encode_special_tokens()
|
||||
}
|
||||
|
||||
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
|
||||
if !skip_special_tokens {
|
||||
return self.tokenizer.decode(ids, false).map_err(Error::Decode);
|
||||
}
|
||||
let filtered_ids: Vec<u32> = ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|id| !self.special_token_ids.contains(id))
|
||||
.collect();
|
||||
self.tokenizer
|
||||
.decode(&filtered_ids, true)
|
||||
.map_err(Error::Decode)
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str {
|
||||
"huggingface"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn codecs_round_trip_and_skip_special_tokens() {
|
||||
let json = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
|
||||
));
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
let ids = tokenizer.encode("<SOS>hello<EOT>").unwrap();
|
||||
|
||||
assert!(tokenizer.decode(&ids, false).unwrap().contains("<SOS>"));
|
||||
assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_filters_special_added_tokens() {
|
||||
let json = r#"{
|
||||
"version": "1.0",
|
||||
"truncation": null,
|
||||
"padding": null,
|
||||
"added_tokens": [
|
||||
{
|
||||
"id": 1,
|
||||
"content": "<s>",
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false,
|
||||
"special": true
|
||||
}
|
||||
],
|
||||
"normalizer": null,
|
||||
"pre_tokenizer": {"type": "Whitespace"},
|
||||
"post_processor": null,
|
||||
"decoder": null,
|
||||
"model": {
|
||||
"type": "WordLevel",
|
||||
"vocab": {"<unk>": 0, "<s>": 1, "hello": 2},
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
}"#;
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
|
||||
assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("<s>"));
|
||||
assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("<s>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vocabulary_lookups_mirror_the_tokenizers_api() {
|
||||
let json = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
|
||||
));
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
let ids = tokenizer.encode("hello").unwrap();
|
||||
|
||||
let token = tokenizer.id_to_token(ids[0]).unwrap();
|
||||
assert_eq!(tokenizer.token_to_id(&token), Some(ids[0]));
|
||||
assert_eq!(tokenizer.id_to_token(u32::MAX), None);
|
||||
assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true));
|
||||
assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false));
|
||||
let added = tokenizer.added_tokens_decoder();
|
||||
assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0));
|
||||
assert!(added.iter().any(|(_, token)| token.special));
|
||||
assert!(tokenizer.padding().is_none());
|
||||
assert!(tokenizer.truncation().is_none());
|
||||
assert!(!tokenizer.encode_special_tokens());
|
||||
assert_eq!(
|
||||
tokenizer.num_special_tokens_to_add(false),
|
||||
tokenizer.encode("").unwrap().len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,5 +6,8 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
once_cell = "1.21.3"
|
||||
rustc-hash = "2.1.3"
|
||||
thiserror.workspace = true
|
||||
tiktoken-rs.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,27 +1,125 @@
|
|||
#![forbid(unsafe_code)]
|
||||
|
||||
mod error;
|
||||
mod ranks;
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
pub use error::UnsupportedTokenizer;
|
||||
pub use ranks::{LoadError, Vocabulary};
|
||||
|
||||
pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE);
|
||||
pub struct TiktokenTokenizer {
|
||||
encoder: &'static tiktoken_rs::CoreBPE,
|
||||
/// Present for encodings built from a rank file; the embedded tiktoken-rs singletons
|
||||
/// behind [`from_name`](Self::from_name) keep their ranks private.
|
||||
vocabulary: Option<&'static Vocabulary>,
|
||||
name: &'static str,
|
||||
}
|
||||
|
||||
impl TiktokenTokenizer {
|
||||
/// Builds `name` from its packaged rank file (read through `load`), once per process.
|
||||
/// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does.
|
||||
pub fn from_cached_ranks(
|
||||
name: &str,
|
||||
load: impl FnOnce(&str) -> std::io::Result<String>,
|
||||
) -> Result<Self, LoadError> {
|
||||
let (loaded, name) = ranks::load(name, load)?;
|
||||
Ok(Self {
|
||||
encoder: &loaded.bpe,
|
||||
vocabulary: Some(&loaded.vocabulary),
|
||||
name,
|
||||
})
|
||||
}
|
||||
|
||||
/// The encodings tiktoken-rs embeds, for hosts without the packaged rank files.
|
||||
pub fn from_name(name: &str) -> Result<Self, UnsupportedTokenizer> {
|
||||
let tokenizer = match name {
|
||||
"cl100k_base" => tiktoken_rs::cl100k_base_singleton(),
|
||||
"o200k_base" => tiktoken_rs::o200k_base_singleton(),
|
||||
"o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(),
|
||||
"p50k_base" => tiktoken_rs::p50k_base_singleton(),
|
||||
"p50k_edit" => tiktoken_rs::p50k_edit_singleton(),
|
||||
"r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(),
|
||||
let (encoder, name) = match name {
|
||||
"cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"),
|
||||
"o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"),
|
||||
"o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"),
|
||||
"p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"),
|
||||
"p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"),
|
||||
"r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"),
|
||||
"gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"),
|
||||
_ => return Err(UnsupportedTokenizer(name.to_owned())),
|
||||
};
|
||||
Ok(Self(tokenizer))
|
||||
Ok(Self {
|
||||
encoder,
|
||||
vocabulary: None,
|
||||
name,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn vocabulary(&self) -> Option<&Vocabulary> {
|
||||
self.vocabulary
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> usize {
|
||||
self.0.count_ordinary(text)
|
||||
self.encoder.count_ordinary(text)
|
||||
}
|
||||
|
||||
pub fn encode(&self, text: &str) -> Vec<u32> {
|
||||
self.encoder.encode_ordinary(text)
|
||||
}
|
||||
|
||||
pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result<Vec<u32>, String> {
|
||||
let allowed = allowed.iter().map(String::as_str).collect();
|
||||
self.encoder
|
||||
.encode(text, &allowed)
|
||||
.map(|(ids, _)| ids)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn special_tokens(&self) -> HashSet<String> {
|
||||
self.encoder
|
||||
.special_tokens()
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every
|
||||
/// token sequence the unstable tail could still become, sorted for a stable order.
|
||||
pub fn encode_with_unstable(
|
||||
&self,
|
||||
text: &str,
|
||||
allowed: &[String],
|
||||
) -> (Vec<u32>, Vec<Vec<u32>>) {
|
||||
let allowed = allowed.iter().map(String::as_str).collect();
|
||||
let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed);
|
||||
let mut completions: Vec<Vec<u32>> = completions.into_iter().collect();
|
||||
completions.sort_unstable();
|
||||
(stable, completions)
|
||||
}
|
||||
|
||||
pub fn decode_bytes(&self, ids: &[u32]) -> Result<Vec<u8>, String> {
|
||||
self.encoder
|
||||
.decode_bytes(ids)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn decode(&self, ids: &[u32]) -> Result<String, String> {
|
||||
self.encoder
|
||||
.decode_bytes(ids)
|
||||
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str {
|
||||
self.name
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encoding_for_model(model: &str) -> Option<&'static str> {
|
||||
match tiktoken_rs::tokenizer::get_tokenizer(model)? {
|
||||
tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => {
|
||||
Some("r50k_base")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -66,5 +164,75 @@ mod tests {
|
|||
panic!("unknown encoding must be rejected");
|
||||
};
|
||||
assert_eq!(name, "unknown-encoding");
|
||||
assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codecs_round_trip_named_encodings() {
|
||||
let encodings = [
|
||||
"cl100k_base",
|
||||
"o200k_base",
|
||||
"o200k_harmony",
|
||||
"p50k_base",
|
||||
"p50k_edit",
|
||||
"r50k_base",
|
||||
"gpt2",
|
||||
];
|
||||
let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"];
|
||||
for name in encodings {
|
||||
let tokenizer = TiktokenTokenizer::from_name(name).unwrap();
|
||||
for text in texts {
|
||||
assert_eq!(
|
||||
tokenizer.decode(&tokenizer.encode(text)).unwrap(),
|
||||
text,
|
||||
"{name}: {text:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoding_token_prefixes_replaces_incomplete_utf8() {
|
||||
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
|
||||
let reference = tiktoken_rs::cl100k_base_singleton();
|
||||
let ids = tokenizer.encode("🙂漢字");
|
||||
for end in 1..=ids.len() {
|
||||
let bytes = reference.decode_bytes(&ids[..end]).unwrap();
|
||||
assert_eq!(
|
||||
tokenizer.decode(&ids[..end]).unwrap(),
|
||||
String::from_utf8_lossy(&bytes),
|
||||
);
|
||||
}
|
||||
assert!(tokenizer.decode(&[u32::MAX]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() {
|
||||
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
|
||||
let text = "hello fanta";
|
||||
let (stable, completions) = tokenizer.encode_with_unstable(text, &[]);
|
||||
assert!(
|
||||
text.as_bytes()
|
||||
.starts_with(&tokenizer.decode_bytes(&stable).unwrap())
|
||||
);
|
||||
assert!(!completions.is_empty());
|
||||
for completion in &completions {
|
||||
let mut ids = stable.clone();
|
||||
ids.extend(completion);
|
||||
assert!(
|
||||
tokenizer
|
||||
.decode_bytes(&ids)
|
||||
.unwrap()
|
||||
.starts_with(text.as_bytes())
|
||||
);
|
||||
}
|
||||
assert!(completions.windows(2).all(|pair| pair[0] < pair[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoding_for_model_maps_known_models() {
|
||||
assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base"));
|
||||
assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base"));
|
||||
assert_eq!(encoding_for_model("unknown-model"), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
340
litellm-rust/crates/token-counter-tiktoken/src/ranks.rs
Normal file
340
litellm-rust/crates/token-counter-tiktoken/src/ranks.rs
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use once_cell::sync::OnceCell;
|
||||
use rustc_hash::FxHashMap;
|
||||
use thiserror::Error;
|
||||
use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank};
|
||||
|
||||
use crate::UnsupportedTokenizer;
|
||||
|
||||
const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4";
|
||||
const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790";
|
||||
const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15";
|
||||
const LEGACY_PATTERN: &str =
|
||||
r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s";
|
||||
const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s";
|
||||
|
||||
static CL100K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static O200K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static HARMONY_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static P50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static EDIT_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static R50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
|
||||
/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built
|
||||
/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does
|
||||
/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`).
|
||||
pub(super) struct Loaded {
|
||||
pub(super) bpe: CoreBPE,
|
||||
pub(super) vocabulary: Vocabulary,
|
||||
}
|
||||
|
||||
/// The byte-level vocabulary of a tiktoken encoding.
|
||||
pub struct Vocabulary {
|
||||
ranks: FxHashMap<Vec<u8>, Rank>,
|
||||
special_tokens: FxHashMap<String, Rank>,
|
||||
max_token_value: Rank,
|
||||
}
|
||||
|
||||
impl Vocabulary {
|
||||
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
|
||||
pub fn token_byte_values(&self) -> Vec<Vec<u8>> {
|
||||
let mut values: Vec<Vec<u8>> = self.ranks.keys().cloned().collect();
|
||||
values.sort_unstable();
|
||||
values
|
||||
}
|
||||
|
||||
/// The rank of one whole token: a mergeable piece first, then a special token's text.
|
||||
pub fn encode_single_token(&self, piece: &[u8]) -> Option<Rank> {
|
||||
if let Some(rank) = self.ranks.get(piece) {
|
||||
return Some(*rank);
|
||||
}
|
||||
std::str::from_utf8(piece)
|
||||
.ok()
|
||||
.and_then(|text| self.special_tokens.get(text).copied())
|
||||
}
|
||||
|
||||
pub fn max_token_value(&self) -> Rank {
|
||||
self.max_token_value
|
||||
}
|
||||
|
||||
/// Every mergeable token with its rank, for building other tables from one parse.
|
||||
pub fn ranks(&self) -> impl Iterator<Item = (&[u8], Rank)> + '_ {
|
||||
self.ranks
|
||||
.iter()
|
||||
.map(|(bytes, rank)| (bytes.as_slice(), *rank))
|
||||
}
|
||||
|
||||
/// The special tokens with their ranks, tiktoken's `_special_tokens`.
|
||||
pub fn special_tokens(&self) -> impl Iterator<Item = (&str, Rank)> + '_ {
|
||||
self.special_tokens
|
||||
.iter()
|
||||
.map(|(token, rank)| (token.as_str(), *rank))
|
||||
}
|
||||
|
||||
pub fn is_special_token(&self, rank: Rank) -> bool {
|
||||
self.special_tokens.values().any(|special| *special == rank)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LoadError {
|
||||
#[error(transparent)]
|
||||
Unsupported(#[from] UnsupportedTokenizer),
|
||||
#[error("failed to load tiktoken ranks: {0}")]
|
||||
Ranks(String),
|
||||
}
|
||||
|
||||
/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays
|
||||
/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share
|
||||
/// one cached encoder.
|
||||
pub(super) fn load(
|
||||
name: &str,
|
||||
load_file: impl FnOnce(&str) -> std::io::Result<String>,
|
||||
) -> Result<(&'static Loaded, &'static str), LoadError> {
|
||||
let (requested, canonical, file, cache) = match name {
|
||||
"cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER),
|
||||
"o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER),
|
||||
"o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER),
|
||||
"p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER),
|
||||
"p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER),
|
||||
"r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER),
|
||||
"gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER),
|
||||
_ => return Err(UnsupportedTokenizer(name.to_owned()).into()),
|
||||
};
|
||||
let loaded = cache.get_or_try_init(|| {
|
||||
let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
build(canonical, &ranks)
|
||||
})?;
|
||||
Ok((loaded, requested))
|
||||
}
|
||||
|
||||
fn build(name: &str, ranks: &str) -> Result<Loaded, LoadError> {
|
||||
let parsed = ranks
|
||||
.lines()
|
||||
.map(parse_rank)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let encoder: FxHashMap<_, _> = parsed
|
||||
.into_iter()
|
||||
.filter(|(_, rank)| name != "r50k_base" || *rank < 50256)
|
||||
.collect();
|
||||
if encoder
|
||||
.values()
|
||||
.collect::<std::collections::HashSet<_>>()
|
||||
.len()
|
||||
!= encoder.len()
|
||||
|| (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..]))
|
||||
{
|
||||
return Err(LoadError::Ranks("invalid vocabulary ranks".into()));
|
||||
}
|
||||
let (pattern, specials): (&str, &[(&str, Rank)]) = match name {
|
||||
"cl100k_base" => (
|
||||
CL100K_PATTERN,
|
||||
&[
|
||||
("<|endoftext|>", 100257),
|
||||
("<|fim_prefix|>", 100258),
|
||||
("<|fim_middle|>", 100259),
|
||||
("<|fim_suffix|>", 100260),
|
||||
("<|endofprompt|>", 100276),
|
||||
],
|
||||
),
|
||||
"o200k_base" => (
|
||||
O200K_BASE_PAT_STR,
|
||||
&[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)],
|
||||
),
|
||||
"o200k_harmony" => (
|
||||
O200K_BASE_PAT_STR,
|
||||
&[
|
||||
("<|startoftext|>", 199998),
|
||||
("<|endoftext|>", 199999),
|
||||
("<|reserved_200000|>", 200000),
|
||||
("<|reserved_200001|>", 200001),
|
||||
("<|return|>", 200002),
|
||||
("<|constrain|>", 200003),
|
||||
("<|reserved_200004|>", 200004),
|
||||
("<|channel|>", 200005),
|
||||
("<|start|>", 200006),
|
||||
("<|end|>", 200007),
|
||||
("<|message|>", 200008),
|
||||
("<|reserved_200009|>", 200009),
|
||||
("<|reserved_200010|>", 200010),
|
||||
("<|reserved_200011|>", 200011),
|
||||
("<|call|>", 200012),
|
||||
],
|
||||
),
|
||||
"p50k_edit" => (
|
||||
LEGACY_PATTERN,
|
||||
&[
|
||||
("<|endoftext|>", 50256),
|
||||
("<|fim_prefix|>", 50281),
|
||||
("<|fim_middle|>", 50282),
|
||||
("<|fim_suffix|>", 50283),
|
||||
],
|
||||
),
|
||||
_ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]),
|
||||
};
|
||||
let reserved = (200013..=201087)
|
||||
.filter(|_| name == "o200k_harmony")
|
||||
.map(|rank| (format!("<|reserved_{rank}|>"), rank));
|
||||
let special_tokens: FxHashMap<String, Rank> = specials
|
||||
.iter()
|
||||
.map(|(token, rank)| ((*token).to_owned(), *rank))
|
||||
.chain(reserved)
|
||||
.collect();
|
||||
let max_token_value = encoder
|
||||
.values()
|
||||
.chain(special_tokens.values())
|
||||
.copied()
|
||||
.max()
|
||||
.ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?;
|
||||
let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern)
|
||||
.map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
Ok(Loaded {
|
||||
bpe,
|
||||
vocabulary: Vocabulary {
|
||||
ranks: encoder,
|
||||
special_tokens,
|
||||
max_token_value,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_rank(line: &str) -> Result<(Vec<u8>, Rank), LoadError> {
|
||||
let (token, rank) = line
|
||||
.split_once(' ')
|
||||
.ok_or_else(|| LoadError::Ranks("missing rank".into()))?;
|
||||
let bytes = STANDARD
|
||||
.decode(token)
|
||||
.map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
let rank = rank
|
||||
.parse()
|
||||
.map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?;
|
||||
Ok((bytes, rank))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::TiktokenTokenizer;
|
||||
|
||||
fn read_packaged_ranks(file: &str) -> std::io::Result<String> {
|
||||
std::fs::read_to_string(
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../../litellm/litellm_core_utils/tokenizers")
|
||||
.join(file),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() {
|
||||
for name in [
|
||||
"cl100k_base",
|
||||
"o200k_base",
|
||||
"o200k_harmony",
|
||||
"p50k_base",
|
||||
"p50k_edit",
|
||||
"r50k_base",
|
||||
"gpt2",
|
||||
] {
|
||||
if name != "gpt2" {
|
||||
assert!(
|
||||
TiktokenTokenizer::from_cached_ranks(name, |_| {
|
||||
Err(std::io::Error::other("unreadable vocabulary"))
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let loads = std::sync::atomic::AtomicUsize::new(0);
|
||||
let barrier = std::sync::Barrier::new(4);
|
||||
let encoders = std::thread::scope(|scope| {
|
||||
let tasks: Vec<_> = (0..4)
|
||||
.map(|_| {
|
||||
scope.spawn(|| {
|
||||
barrier.wait();
|
||||
TiktokenTokenizer::from_cached_ranks(name, |file| {
|
||||
loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
read_packaged_ranks(file)
|
||||
})
|
||||
.unwrap()
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
tasks
|
||||
.into_iter()
|
||||
.map(|task| task.join().unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
assert_eq!(loads.into_inner(), usize::from(name != "gpt2"));
|
||||
let actual = &encoders[0];
|
||||
let expected = TiktokenTokenizer::from_name(name).unwrap();
|
||||
assert_eq!(actual.special_tokens(), expected.special_tokens());
|
||||
let specials: Vec<_> = expected.special_tokens().into_iter().collect();
|
||||
let special_text = specials.join(" ");
|
||||
assert_eq!(
|
||||
actual.encode_special(&special_text, &specials).unwrap(),
|
||||
expected.encode_special(&special_text, &specials).unwrap()
|
||||
);
|
||||
for text in [
|
||||
"",
|
||||
"café 漢字 ع 🙂",
|
||||
"a\r\nb\t ",
|
||||
" hello 123456789",
|
||||
&special_text,
|
||||
] {
|
||||
let ids = expected.encode(text);
|
||||
assert_eq!(actual.encode(text), ids, "{name}: {text:?}");
|
||||
assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}");
|
||||
assert_eq!(
|
||||
actual.decode_bytes(&ids).unwrap(),
|
||||
expected.decode_bytes(&ids).unwrap()
|
||||
);
|
||||
}
|
||||
let cached = TiktokenTokenizer::from_cached_ranks(name, |_| {
|
||||
panic!("reloaded cached vocabulary")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(cached.encode("cached"), expected.encode("cached"));
|
||||
assert_eq!(cached.name(), name);
|
||||
assert!(expected.vocabulary().is_none());
|
||||
assert_vocabulary_lookups(name, actual);
|
||||
}
|
||||
}
|
||||
|
||||
/// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the
|
||||
/// encoder itself and against the known vocabulary sizes.
|
||||
fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) {
|
||||
let max_token_value = match name {
|
||||
"cl100k_base" => 100_276,
|
||||
"o200k_base" => 200_018,
|
||||
"o200k_harmony" => 201_087,
|
||||
"p50k_base" => 50_280,
|
||||
"p50k_edit" => 50_283,
|
||||
"r50k_base" | "gpt2" => 50_256,
|
||||
_ => unreachable!("{name}"),
|
||||
};
|
||||
let vocabulary = tokenizer.vocabulary().unwrap();
|
||||
assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}");
|
||||
let values = vocabulary.token_byte_values();
|
||||
assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}");
|
||||
for piece in values.iter().step_by(997) {
|
||||
let rank = vocabulary.encode_single_token(piece).unwrap();
|
||||
assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}");
|
||||
assert!(!vocabulary.is_special_token(rank), "{name}");
|
||||
}
|
||||
for (token, rank) in vocabulary.special_tokens() {
|
||||
assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank));
|
||||
assert!(vocabulary.is_special_token(rank), "{name}: {token}");
|
||||
}
|
||||
assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_ranks_return_errors_instead_of_panicking() {
|
||||
for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] {
|
||||
assert!(build("cl100k_base", ranks).is_err());
|
||||
}
|
||||
let repeated_rank = (0..=u8::MAX)
|
||||
.map(|byte| format!("{} 0\n", STANDARD.encode([byte])))
|
||||
.collect::<String>();
|
||||
assert!(build("cl100k_base", &repeated_rank).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,10 @@
|
|||
# Token counting
|
||||
|
||||
`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface
|
||||
`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer`
|
||||
|
||||
Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing
|
||||
|
||||
`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it
|
||||
|
||||
The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation
|
||||
|
||||
|
|
@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t
|
|||
|
||||
The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2`
|
||||
|
||||
All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
|
||||
All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
|
||||
|
||||
Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend
|
||||
|
||||
Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ pub enum Error {
|
|||
JsonUtf8(#[source] FromUtf8Error),
|
||||
#[error("tokenization failed: {0}")]
|
||||
Encode(#[source] Box<dyn std::error::Error + Send + Sync>),
|
||||
#[error("token decoding failed: {0}")]
|
||||
Decode(String),
|
||||
#[error("token counting task failed: {0}")]
|
||||
Task(String),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,31 @@
|
|||
use litellm_token_counter_fast::Error as BackendError;
|
||||
pub use litellm_token_counter_fast::FastTokenizer;
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
pub trait FastCounter: TextCodec {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
impl FastCounter for crate::huggingface::HuggingFaceTokenizer {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
Some(FastTokenizer::from_shared(self.shared()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
impl FastCounter for crate::tiktoken::TiktokenTokenizer {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
let vocabulary = self.vocabulary()?;
|
||||
match self.name() {
|
||||
"cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()),
|
||||
"o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()),
|
||||
_ => return None,
|
||||
}
|
||||
.ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_json_fast(tokenizer_json: &str) -> Result<Self, Error> {
|
||||
|
|
@ -39,3 +63,65 @@ impl From<BackendError> for Error {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::huggingface::HuggingFaceTokenizer;
|
||||
use crate::tiktoken::TiktokenTokenizer;
|
||||
|
||||
const TEXTS: [&str; 4] = [
|
||||
"",
|
||||
"hello world <|endoftext|>",
|
||||
"café 漢字 ع 🙂 line\r\n indented 123456789",
|
||||
"<SOS>system<EOT> a\u{301} fi",
|
||||
];
|
||||
|
||||
fn packaged(file: &str) -> String {
|
||||
std::fs::read_to_string(format!(
|
||||
"{}/../../../litellm/litellm_core_utils/tokenizers/{file}",
|
||||
env!("CARGO_MANIFEST_DIR")
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fast_counters_derived_from_codecs_count_like_the_codecs() {
|
||||
let huggingface =
|
||||
HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap();
|
||||
let fast = huggingface.fast_counter().unwrap();
|
||||
for text in TEXTS {
|
||||
assert_eq!(
|
||||
fast.count_tokens(text).unwrap(),
|
||||
Tokenizer::count_tokens(&huggingface, text).unwrap(),
|
||||
"{text:?}"
|
||||
);
|
||||
}
|
||||
|
||||
for name in ["cl100k_base", "o200k_base", "o200k_harmony"] {
|
||||
let tiktoken =
|
||||
TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap();
|
||||
let fast = tiktoken.fast_counter().unwrap();
|
||||
for text in TEXTS {
|
||||
assert_eq!(
|
||||
fast.count_tokens(text).unwrap(),
|
||||
tiktoken.count_tokens(text),
|
||||
"{name}: {text:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodings_without_a_fast_scanner_keep_the_codec() {
|
||||
let tiktoken =
|
||||
TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap();
|
||||
assert!(tiktoken.fast_counter().is_none());
|
||||
assert!(
|
||||
TiktokenTokenizer::from_name("cl100k_base")
|
||||
.unwrap()
|
||||
.fast_counter()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
use litellm_token_counter_huggingface::Error as BackendError;
|
||||
pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
|
||||
pub use litellm_token_counter_huggingface::{
|
||||
AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection,
|
||||
PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json,
|
||||
encoding_to_json,
|
||||
};
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
|
||||
|
|
@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
impl TextCodec for HuggingFaceTokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
HuggingFaceTokenizer::encode(self, text).map_err(Error::from)
|
||||
}
|
||||
|
||||
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
|
||||
HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from)
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
HuggingFaceTokenizer::name(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<BackendError> for Error {
|
||||
fn from(error: BackendError) -> Self {
|
||||
match error {
|
||||
BackendError::Load(source) => Self::Load(source),
|
||||
BackendError::Encode(source) => Self::Encode(source),
|
||||
BackendError::Decode(source) => Self::Decode(source.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,5 +20,5 @@ pub mod tiktoken;
|
|||
|
||||
pub use counter::{InputTokenCount, TokenCounter};
|
||||
pub use error::Error;
|
||||
pub use tokenizer::Tokenizer;
|
||||
pub use tokenizer::{TextCodec, Tokenizer};
|
||||
pub use types::CountableRequest;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
pub use litellm_token_counter_tiktoken::TiktokenTokenizer;
|
||||
use litellm_token_counter_tiktoken::UnsupportedTokenizer;
|
||||
use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer};
|
||||
pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model};
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_tiktoken(encoding: &str) -> Result<Self, Error> {
|
||||
|
|
@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
impl TextCodec for TiktokenTokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
Ok(TiktokenTokenizer::encode(self, text))
|
||||
}
|
||||
|
||||
fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result<String, Error> {
|
||||
TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string()))
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
TiktokenTokenizer::name(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<UnsupportedTokenizer> for Error {
|
||||
fn from(error: UnsupportedTokenizer) -> Self {
|
||||
Self::UnsupportedTokenizer(error.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<LoadError> for Error {
|
||||
fn from(error: LoadError) -> Self {
|
||||
match error {
|
||||
LoadError::Unsupported(error) => error.into(),
|
||||
LoadError::Ranks(message) => Self::Ranks(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync {
|
|||
fn count_tokens(&self, text: &str) -> Result<usize, Error>;
|
||||
}
|
||||
|
||||
pub trait TextCodec: Tokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error>;
|
||||
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error>;
|
||||
fn name(&self) -> &str;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -58,7 +58,8 @@ from ._lazy_imports_registry import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
from tiktoken import Encoding
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
def get_litellm_globals() -> dict[str, object]:
|
||||
|
|
@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
|
|||
# These are special lazy loaders for things that are used internally
|
||||
# They're separate from the main lazy import system because they have specific use cases
|
||||
|
||||
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
|
||||
_default_encoding: "Encoding | None" = None
|
||||
|
||||
def _get_default_encoding() -> "Tokenizer":
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
|
||||
def _get_default_encoding() -> "Encoding":
|
||||
"""
|
||||
Lazily load and cache the default OpenAI encoding.
|
||||
|
||||
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
|
||||
at `litellm` import time. The encoding is cached after the first import.
|
||||
|
||||
This is used internally by utils.py functions that need the encoding but shouldn't
|
||||
trigger its import during module load.
|
||||
"""
|
||||
global _default_encoding
|
||||
if _default_encoding is None:
|
||||
from litellm.litellm_core_utils.default_encoding import encoding
|
||||
|
||||
_default_encoding = encoding
|
||||
return _default_encoding
|
||||
return get_encoding("cl100k_base")
|
||||
|
||||
|
||||
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from types import MappingProxyType
|
|||
from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable
|
||||
|
||||
from litellm import main
|
||||
from litellm.rust_bridge.catalog import Context, Delivery, Route
|
||||
from litellm.rust_bridge.catalog import Delivery, Route, RouteContext
|
||||
from litellm.rust_bridge.chat_completions.entrypoints import (
|
||||
NATIVE_ACOMPLETION,
|
||||
NATIVE_COMPLETION,
|
||||
|
|
@ -72,8 +72,8 @@ def _public_request(
|
|||
)
|
||||
|
||||
|
||||
def _context(request: LiteLLMChatCompletionsRequest) -> Context:
|
||||
return Context(
|
||||
def _context(request: LiteLLMChatCompletionsRequest) -> RouteContext:
|
||||
return RouteContext(
|
||||
Route.CHAT_COMPLETIONS,
|
||||
provider=request.custom_llm_provider,
|
||||
model=request.model,
|
||||
|
|
|
|||
|
|
@ -323,6 +323,9 @@ class OCRPricing(TypedDict, total=False):
|
|||
annotation_cost_per_page: ReadOnly[float | None]
|
||||
|
||||
|
||||
_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"})
|
||||
|
||||
|
||||
def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool:
|
||||
return (
|
||||
(model_info.get("input_cost_per_token") or 0.0) > 0
|
||||
|
|
@ -331,6 +334,11 @@ def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool:
|
||||
mode: Final = model_info.get("mode")
|
||||
return mode is None or mode in _WALL_CLOCK_PRICED_MODES
|
||||
|
||||
|
||||
def _per_second_pricing_cost(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
|
|
@ -340,7 +348,7 @@ def _per_second_pricing_cost(
|
|||
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
|
||||
return None
|
||||
if _has_token_or_tiered_pricing(model_info):
|
||||
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
|
||||
return None
|
||||
input_cost_per_second: Final = model_info.get("input_cost_per_second")
|
||||
output_cost_per_second: Final = model_info.get("output_cost_per_second")
|
||||
|
|
|
|||
|
|
@ -427,7 +427,7 @@ class LLMCallSpanData:
|
|||
# plain ``.get`` — no repeated ``isinstance`` guards.
|
||||
raw_response: Final = payload.get("response")
|
||||
response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {})
|
||||
choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response)
|
||||
choices_out: Final = _output_choices(response)
|
||||
# ``finish_reasons`` is metadata, not content, so derive it from
|
||||
# ``choices_out`` before gating. The raw message/choice bodies are only
|
||||
# retained when content capture is enabled (see ``capture_span_content``);
|
||||
|
|
@ -752,20 +752,99 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
|||
return (choice,)
|
||||
|
||||
|
||||
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
markdowns: Final = tuple(
|
||||
text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None
|
||||
def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
"""The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes."""
|
||||
return (
|
||||
_completion_choices(response)
|
||||
or _responses_choices(response)
|
||||
or _ocr_choices(response)
|
||||
or _transcription_choices(response)
|
||||
or _moderation_choices(response)
|
||||
or _image_choices(response)
|
||||
or _binary_choices(response)
|
||||
)
|
||||
if not markdowns:
|
||||
|
||||
|
||||
def _text_choice(content: str, finish_reason: str | None = None) -> _Choice:
|
||||
message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}
|
||||
return {"message": message, "finish_reason": finish_reason}
|
||||
|
||||
|
||||
def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]:
|
||||
return (_text_choice("\n\n".join(parts)),) if parts else ()
|
||||
|
||||
|
||||
def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
_text_choice(text, as_str(choice.get("finish_reason")))
|
||||
if "message" not in choice and isinstance(text := choice.get("text"), str)
|
||||
else choice
|
||||
for choice in _dicts(response.get("choices"))
|
||||
)
|
||||
|
||||
|
||||
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None)
|
||||
)
|
||||
|
||||
|
||||
def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
text: Final = response.get("text")
|
||||
return (_text_choice(text),) if isinstance(text, str) and text else ()
|
||||
|
||||
|
||||
def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(
|
||||
_moderation_verdict(flagged, result.get("categories"))
|
||||
for result in _dicts(response.get("results"))
|
||||
if isinstance(flagged := result.get("flagged"), bool)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _moderation_verdict(flagged: bool, categories: object) -> str:
|
||||
if not flagged:
|
||||
return "not flagged"
|
||||
hits: Final = (
|
||||
tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True)
|
||||
if isinstance(categories, dict)
|
||||
else ()
|
||||
)
|
||||
return f"flagged: {', '.join(hits)}" if hits else "flagged"
|
||||
|
||||
|
||||
def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None)
|
||||
)
|
||||
|
||||
|
||||
def _image_summary(item: Mapping[str, object]) -> str | None:
|
||||
location: Final = _image_location(item)
|
||||
if location is None:
|
||||
return None
|
||||
revised: Final = as_str(item.get("revised_prompt"))
|
||||
return f"{revised}\n{location}" if revised else location
|
||||
|
||||
|
||||
def _image_location(item: Mapping[str, object]) -> str | None:
|
||||
url: Final = as_str(item.get("url"))
|
||||
if url is not None:
|
||||
return url
|
||||
encoded: Final = item.get("b64_json")
|
||||
if not isinstance(encoded, str):
|
||||
return None
|
||||
return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)"
|
||||
|
||||
|
||||
def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
size: Final = as_int(response.get("num_bytes"))
|
||||
if size is None:
|
||||
return ()
|
||||
message: Final[_AssistantMessage] = {
|
||||
"role": "assistant",
|
||||
"content": "\n\n".join(markdowns),
|
||||
"refusal": None,
|
||||
"tool_calls": None,
|
||||
}
|
||||
choice: Final[_Choice] = {"message": message, "finish_reason": None}
|
||||
return (choice,)
|
||||
content_type: Final = as_str(response.get("content_type"))
|
||||
return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),)
|
||||
|
||||
|
||||
def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None:
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@ Core files:
|
|||
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
|
||||
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
|
||||
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
|
||||
- `default_encoding.py`: code for loading the default encoding (tiktoken)
|
||||
- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache
|
||||
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
|
||||
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
|
||||
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
|
||||
|
||||
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue