Merge remote-tracking branch 'origin/main' into litellm_spend-logs-end-date

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

# Conflicts:
#	tests/e2e/proxy_client.py
This commit is contained in:
kerry 2026-09-22 17:35:34 +00:00
commit 0d5ccb6d5e
317 changed files with 16268 additions and 2249 deletions

View file

@ -15,6 +15,12 @@ description: >-
cache the same directory for different workloads, and a shared key would let
whichever ran first deny the others a save.
inputs:
profile:
description: "Cargo profile the build uses (dev or release)"
required: false
default: "dev"
runs:
using: composite
steps:
@ -25,6 +31,6 @@ runs:
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-maturin-dev-
${{ runner.os }}-maturin-${{ inputs.profile }}-

View file

@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError
secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
MIN_MASKED_LENGTH: Final = 8
ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
def main() -> int:
@ -30,10 +31,15 @@ def main() -> int:
f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
)
return 1
for value in secrets.values():
if len(value) >= MIN_MASKED_LENGTH:
_ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
sys.stdout.flush()
if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
_ = sys.stdout.write(
"".join(
f"::add-mask::{value.replace('%', '%25')}\n"
for value in secrets.values()
if len(value) >= MIN_MASKED_LENGTH
)
)
sys.stdout.flush()
lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
try:
with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:

View file

@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile(
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
)
HARNESS: Final = re.compile(
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"

View file

@ -134,7 +134,16 @@ def main(
uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
native_path.parent.mkdir(parents=True, exist_ok=True)
native_path.write_bytes(archive.read(native_member))
native_bytes: Final = archive.read(native_member)
native_path.write_bytes(native_bytes)
duplicated_vocabularies: Final = tuple(
member.filename
for member in wheel_members
if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
and member.file_size > 0
and archive.read(member) in native_bytes
)
wheel_metadata_tags_match: Final = (
len(wheel_metadata_tags) == len(expanded_filename_tags)
@ -205,7 +214,7 @@ def main(
native_module: Final = load_native_module(native_path)
native_module_loads: Final = native_module is not None
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
native_size_limit: Final = 40_000_000
native_size_limit: Final = 35_000_000
native_size_within_limit: Final = native_member.file_size <= native_size_limit
validations: Final = (
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@ -223,6 +232,7 @@ def main(
("Native module loads", native_module_loads),
("Production module omits the panic test hook", panic_test_hook_absent),
(f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
("Wheel contents are valid", not unexpected_members),
)

View file

@ -13,6 +13,7 @@ on:
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
- ".github/scripts/uv_sync_with_retries.sh"
pull_request:
branches:
- main
@ -25,6 +26,7 @@ on:
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
- ".github/scripts/uv_sync_with_retries.sh"
# Allow CodSpeed to trigger backtest performance analysis
# in order to generate initial data
workflow_dispatch:
@ -59,19 +61,27 @@ jobs:
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
with:
profile: release
# Build the wheel and resolve every dependency outside the CodSpeed
# runner: the same maturin build took 42 minutes inside `codspeed run`
# versus under 3 minutes as a plain step (LIT-6183)
- name: Build environment
- name: Build the release wheel
run: uv build --wheel --out-dir dist
- name: Install the wheel into the benchmark environment
run: |
UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
- name: Collect benchmarks
env:
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
run: >
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
uv run --frozen --no-default-groups
--with pytest==8.3.5
--with pytest-codspeed==4.3.0
--with "mcp>=2.2.0,<3.0"
--with "a2a-sdk>=1.1.0,<2.0"
pytest
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
--import-mode=importlib
-p pytest_codspeed.plugin
tests/benchmarks/
--codspeed
@ -82,13 +92,9 @@ jobs:
with:
mode: simulation
run: >
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
uv run --frozen --no-default-groups
--with pytest==8.3.5
--with pytest-codspeed==4.3.0
--with "mcp>=2.2.0,<3.0"
--with "a2a-sdk>=1.1.0,<2.0"
pytest
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
--import-mode=importlib
-p pytest_codspeed.plugin
tests/benchmarks/
--codspeed

View file

@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
--extra saml \
--python python3.13
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
npm_config_cache=/root/.npm \
prisma generate --schema=./schema.prisma

View file

@ -1,8 +1,11 @@
"""Guard the cost map on pull requests.
Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
files and may only add or update models.
"""
from __future__ import annotations
@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
)
def touches_cost_map(changed_files: Sequence[str]) -> bool:
return any(path in GUARDED_PATHS for path in changed_files)
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
if bot:
return "bot contract enforced"
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
if not bot and not touches_cost_map(changed_files):
return ()
head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
if isinstance(head_map, str):
return (head_map,)
return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
def _git(*args: str) -> str:
def _git(*args: str) -> str | None:
result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
return result.stdout if result.returncode == 0 else ""
return result.stdout if result.returncode == 0 else None
def snapshot(revision: str) -> Snapshot:
return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
diff: Final = _git("diff", "--name-only", "--no-renames", base, head)
return None if diff is None else tuple(diff.splitlines())
def main(argv: Sequence[str]) -> int:
@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int:
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
args: Final = parser.parse_args(argv)
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
changed: Final = changed_files(args.base, args.head)
if changed is None:
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
return 1
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
contract: Final = contract_for(bot, changed)
if failures:
print(f"cost map guard failed ({contract}):")
print("\n".join(f"- {failure}" for failure in failures))

View file

@ -3065,6 +3065,7 @@ dependencies = [
"serde_json",
"serde_with",
"sha2 0.10.9",
"thiserror 2.0.19",
"tokio",
"tokio-tungstenite",
"url",
@ -3253,6 +3254,7 @@ dependencies = [
name = "litellm-token-counter-huggingface"
version = "0.1.0"
dependencies = [
"serde_json",
"thiserror 2.0.19",
"tokenizers",
]
@ -3261,6 +3263,9 @@ dependencies = [
name = "litellm-token-counter-tiktoken"
version = "0.1.0"
dependencies = [
"base64 0.22.1",
"once_cell",
"rustc-hash",
"thiserror 2.0.19",
"tiktoken-rs",
]

View file

@ -89,7 +89,7 @@ veil = "0.3.0"
[profile.release]
opt-level = 3
lto = "thin"
lto = "fat"
codegen-units = 1
panic = "unwind"
debug = false

View file

@ -29,7 +29,7 @@ pyo3::create_exception!(
static FORK_GATE: ForkGate = ForkGate::new();
/// Whether this process has started the Tokio runtime.
/// Whether this process has entered process-bound native execution.
pub fn runtime_started() -> bool {
FORK_GATE.started(std::process::id())
}
@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> {
FORK_GATE.reserve(std::process::id())
}
/// The only door to the Tokio runtime: every route reaches it through this module, which is
/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it.
fn enter_runtime() -> PyResult<()> {
/// Claims process-bound native state before runtime startup or tokenizer execution.
pub fn enter_native() -> PyResult<()> {
FORK_GATE
.enter(std::process::id())
.map_err(|refused| match refused {
@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> {
#[expect(clippy::disallowed_methods, reason = "this is the gated door")]
fn runtime() -> PyResult<&'static Runtime> {
enter_runtime()?;
enter_native()?;
Ok(pyo3_async_runtimes::tokio::get_runtime())
}
@ -70,7 +69,7 @@ where
F: Future<Output = PyResult<T>> + Send + 'static,
T: for<'py> IntoPyObject<'py> + Send + 'static,
{
enter_runtime()?;
enter_native()?;
pyo3_async_runtimes::tokio::future_into_py(py, future)
}

View file

@ -20,7 +20,7 @@ pub use argument::lookup;
pub use callable::wrap_failure;
pub use driver::run_call;
pub use execution::{
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value,
reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value,
runtime_started,
};

View file

@ -10,7 +10,7 @@ name = "_native"
crate-type = ["cdylib"]
[features]
default = ["abi3", "fast"]
default = ["abi3", "fast", "huggingface", "tiktoken"]
abi3 = ["pyo3/abi3-py310"]
extension-module = ["pyo3/extension-module"]
panic-test = []
@ -52,8 +52,9 @@ pyo3-async-runtimes.workspace = true
reqwest.workspace = true
redis = { version = "1.7.0", features = ["tls-rustls"] }
serde_json.workspace = true
url.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "sync"] }
url.workspace = true
[dev-dependencies]
litellm-secrets-aws.workspace = true

View file

@ -12,7 +12,7 @@ mod routes;
reason = "secret-manager foundations await rollout activation"
)]
mod secrets;
mod token_counter;
mod tokenizer;
#[pymodule(gil_used = true)]
mod _native {
@ -37,7 +37,12 @@ mod _native {
#[pymodule_export]
use crate::routes::responses::ResponsesWebSocketConnection;
#[pymodule_export]
use crate::token_counter::TokenCounter;
use crate::routes::token_counter::TokenCounter;
#[cfg(feature = "huggingface")]
#[pymodule_export]
use crate::tokenizer::HuggingFaceEncoding;
#[pymodule_export]
use crate::tokenizer::Tokenizer;
#[pymodule_export]
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
use pyo3::{prelude::*, types::PyModule};
@ -83,10 +88,13 @@ mod tests {
"achat_completions",
"ResponsesWebSocketConnection",
"TokenCounter",
"Tokenizer",
"gil_stats",
"process_state_started",
"reserve_process_for_forking",
];
#[cfg(feature = "huggingface")]
expected.push("HuggingFaceEncoding");
expected.sort_unstable();
let mut public_names: Vec<String> = native_module(py)

View file

@ -3,6 +3,7 @@ pub(crate) mod chat_completions;
pub(crate) mod messages;
pub(crate) mod ocr;
pub(crate) mod responses;
pub(crate) mod token_counter;
#[cfg(test)]
mod tests {

View file

@ -131,7 +131,7 @@ impl RouteHost for OcrRouteHost {
fn classify(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::callback::python_error(py, source)
&& let Some(original) = crate::secrets::python_error(py, source)
{
return Ok(original);
}

View file

@ -0,0 +1,87 @@
use std::sync::Arc;
use std::{num::NonZero, thread::available_parallelism};
use litellm_host_python::{enter_native, run_async};
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
types::PyAny,
};
use tokio::sync::Semaphore;
use crate::errors::RustBridgeDeclined;
use crate::tokenizer::Tokenizer;
/// Counts the input tokens of a raw request body off the Python event loop with
/// the GIL released. Python owns which requests get here and what to do with
/// the count. At most one encode per core runs at a time; the rest wait in the
/// async task, where a cancelled Python awaiter drops them before any blocking
/// work is scheduled.
#[pyclass(frozen)]
pub(crate) struct TokenCounter {
inner: Arc<CoreTokenCounter>,
encode_slots: Arc<Semaphore>,
}
#[pymethods]
impl TokenCounter {
#[staticmethod]
#[pyo3(signature = (tokenizer, fast = false))]
fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult<Self> {
enter_native()?;
let inner = CoreTokenCounter::new(tokenizer.counter(py, fast));
Ok(Self {
inner: Arc::new(inner),
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
})
}
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
let counter = Arc::clone(&self.inner);
let encode_slots = Arc::clone(&self.encode_slots);
let body = body.to_vec();
run_async(
py,
async move {
let _slot = encode_slots
.acquire_owned()
.await
.map_err(|error| Error::Task(error.to_string()))?;
tokio::task::spawn_blocking(move || count_body(&counter, &body))
.await
.map_err(|error| Error::Task(error.to_string()))?
},
token_count_error_to_pyerr,
)
}
}
fn encode_parallelism() -> usize {
available_parallelism().map_or(1, NonZero::get)
}
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
let request = CountableRequest::parse(body)?;
counter.count_request(&request)
}
pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr {
let message = error.to_string();
match error {
Error::Load(_)
| Error::Ranks(_)
| Error::UnicodeClasses
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
Error::RequestParse(_)
| Error::MissingInput
| Error::FloatText
| Error::ContentBlock
| Error::ArrayItems
| Error::JsonSerialization(_)
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
}
}

View file

@ -1,45 +1,20 @@
use std::{fmt, future::Future, pin::Pin};
use std::{future::Future, pin::Pin};
use litellm_core_utils::settings::Lookup;
use litellm_secrets::{
Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
};
use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
use pyo3::{prelude::*, types::PyDict};
use super::error::external_error;
const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler";
struct PythonSecretError(Py<PyBaseException>);
impl fmt::Debug for PythonSecretError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("PythonSecretError")
}
}
impl fmt::Display for PythonSecretError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("Python secret manager failed")
}
}
impl std::error::Error for PythonSecretError {}
pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option<PyErr> {
let Error::ExternalManager(source) = error else {
return None;
};
source
.downcast_ref::<PythonSecretError>()
.map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any()))
}
/// A secret manager whose reads execute in Python: a custom manager, a legacy compatible
/// client, or a manually assigned SDK client.
pub(crate) struct PythonSecretManager {
client: Py<PyAny>,
system: Option<KeyManagementSystem>,
/// The `key_manager` name Python's handler dispatches on.
key_manager: &'static str,
settings: Option<Py<PyAny>>,
}
@ -52,7 +27,6 @@ impl PythonSecretManager {
Self {
client,
system,
key_manager: system.map_or("local", python_name),
settings,
}
}
@ -78,14 +52,12 @@ impl PythonSecretManager {
}
let kwargs = PyDict::new(py);
kwargs.set_item("client", client)?;
kwargs.set_item("key_manager", self.key_manager)?;
kwargs.set_item("key_manager", self.system.map_or("local", python_name))?;
kwargs.set_item("secret_name", name)?;
kwargs.set_item(
"key_management_settings",
self.settings
.as_ref()
.map_or_else(|| py.None(), |settings| settings.clone_ref(py)),
)?;
match &self.settings {
Some(settings) => kwargs.set_item("key_management_settings", settings.bind(py))?,
None => kwargs.set_item("key_management_settings", py.None())?,
}
py.import(HANDLER_MODULE)?
.getattr("get_secret_from_manager")?
.call((), Some(&kwargs))?
@ -123,9 +95,7 @@ impl ExternalSecretManager for PythonSecretManager {
Python::attach(|py| {
self.read(py, name)
.map(|value| value.map(SecretValue::new).map(Secret::String))
.map_err(|error| {
Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py))))
})
.map_err(|error| external_error(py, error))
})
})
}
@ -140,19 +110,27 @@ mod tests {
SecretManagerState, SecretResolver,
};
use pyo3::{prelude::*, types::PyDict};
use rstest::rstest;
use super::{HANDLER_MODULE, PythonSecretManager, python_error, python_name};
use super::{HANDLER_MODULE, PythonSecretManager, python_name};
use crate::secrets::python_error;
#[rstest]
#[case::value_error("ValueError", None)]
#[case::value_error_with_fallback("ValueError", Some("environment-key"))]
#[case::cancelled("asyncio.CancelledError", None)]
#[case::cancelled_with_fallback("asyncio.CancelledError", Some("environment-key"))]
#[tokio::test]
async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback() {
async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback(
#[case] failure_type: &str,
#[case] fallback: Option<&'static str>,
) {
Python::initialize();
for failure_type in ["ValueError", "asyncio.CancelledError"] {
for fallback in [None, Some("environment-key")] {
let (reader, locals) = Python::attach(|py| {
let locals = PyDict::new(py);
locals.set_item("failure_type", failure_type).unwrap();
py.run(
c"
let (reader, locals) = Python::attach(|py| {
let locals = PyDict::new(py);
locals.set_item("failure_type", failure_type).unwrap();
py.run(
c"
import asyncio
failure = eval(failure_type)('secret manager failed')
cause = RuntimeError('original cause')
@ -164,48 +142,46 @@ class Manager:
raise failure
manager = Manager()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let reader = PythonSecretManager::new(
locals.get_item("manager").unwrap().unwrap().unbind(),
None,
None,
);
(reader, locals.unbind())
});
let resolver = SecretResolver::new(
Arc::new(SecretManagerState::new(
SecretManager::External(Arc::new(reader)),
KeyManagementSettings::default(),
)),
Arc::new(move |_: &str| fallback.map(str::to_owned)),
OidcResolver::default(),
)
.with_failure_policy(FailurePolicy::EnvironmentFallback);
let error = resolver.get_secret("API_KEY", None).await.unwrap_err();
Python::attach(|py| {
let original = python_error(py, &error).unwrap();
let locals = locals.bind(py);
assert!(
original
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] {
assert!(
original
.value(py)
.getattr(attribute)
.unwrap()
.is(locals.get_item(name).unwrap().unwrap())
);
}
assert!(original.traceback(py).is_some());
});
Some(&locals),
Some(&locals),
)
.unwrap();
let reader = PythonSecretManager::new(
locals.get_item("manager").unwrap().unwrap().unbind(),
None,
None,
);
(reader, locals.unbind())
});
let resolver = SecretResolver::new(
Arc::new(SecretManagerState::new(
SecretManager::External(Arc::new(reader)),
KeyManagementSettings::default(),
)),
Arc::new(move |_: &str| fallback.map(str::to_owned)),
OidcResolver::default(),
)
.with_failure_policy(FailurePolicy::EnvironmentFallback);
let error = resolver.get_secret("API_KEY", None).await.unwrap_err();
Python::attach(|py| {
let original = python_error(py, &error).unwrap();
let locals = locals.bind(py);
assert!(
original
.value(py)
.is(locals.get_item("failure").unwrap().unwrap())
);
for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] {
assert!(
original
.value(py)
.getattr(attribute)
.unwrap()
.is(locals.get_item(name).unwrap().unwrap())
);
}
}
assert!(original.traceback(py).is_some());
});
}
/// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and
@ -245,47 +221,57 @@ for name in installed:
.unwrap();
}
#[test]
fn python_names_round_trip_through_serde() {
for system in [
KeyManagementSystem::GoogleKms,
KeyManagementSystem::AzureKeyVault,
KeyManagementSystem::AwsSecretManager,
KeyManagementSystem::GoogleSecretManager,
KeyManagementSystem::HashicorpVault,
KeyManagementSystem::Cyberark,
KeyManagementSystem::Local,
KeyManagementSystem::AwsKms,
KeyManagementSystem::Custom,
] {
assert_eq!(
serde_json::to_value(system).unwrap(),
serde_json::Value::String(python_name(system).to_owned())
);
}
#[rstest]
#[case::google_kms(KeyManagementSystem::GoogleKms)]
#[case::azure_key_vault(KeyManagementSystem::AzureKeyVault)]
#[case::aws_secret_manager(KeyManagementSystem::AwsSecretManager)]
#[case::google_secret_manager(KeyManagementSystem::GoogleSecretManager)]
#[case::hashicorp_vault(KeyManagementSystem::HashicorpVault)]
#[case::cyberark(KeyManagementSystem::Cyberark)]
#[case::local(KeyManagementSystem::Local)]
#[case::aws_kms(KeyManagementSystem::AwsKms)]
#[case::custom(KeyManagementSystem::Custom)]
fn python_names_round_trip_through_serde(#[case] system: KeyManagementSystem) {
assert_eq!(
serde_json::to_value(system).unwrap(),
serde_json::Value::String(python_name(system).to_owned())
);
}
#[test]
fn custom_readers_without_a_system_are_called_directly() {
#[rstest]
#[case::legacy(None, false)]
#[case::custom(Some(KeyManagementSystem::Custom), true)]
fn direct_readers_receive_compatible_kwargs(
#[case] system: Option<KeyManagementSystem>,
#[case] expects_optional_params: bool,
) {
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
py.run(
c"
class Settings:
def model_dump(self):
return {'scope': 'custom'}
class Manager:
def __init__(self):
self.names = []
self.optional_params = []
def sync_read_secret(self, secret_name, optional_params=None, timeout=None):
self.names.append(secret_name)
self.optional_params.append(optional_params)
return 'direct-' + secret_name
manager = Manager()
settings = Settings()
",
Some(&locals),
Some(&locals),
)
.unwrap();
let manager = locals.get_item("manager").unwrap().unwrap();
let reader = PythonSecretManager::new(manager.clone().unbind(), None, None);
let settings = expects_optional_params
.then(|| locals.get_item("settings").unwrap().unwrap().unbind());
let reader = PythonSecretManager::new(manager.clone().unbind(), system, settings);
assert_eq!(
reader.read(py, "API_KEY").unwrap().as_deref(),
Some("direct-API_KEY")
@ -298,6 +284,23 @@ manager = Manager()
.unwrap(),
["API_KEY"]
);
let optional_params = manager
.getattr("optional_params")
.unwrap()
.get_item(0)
.unwrap();
if expects_optional_params {
assert_eq!(
optional_params
.get_item("scope")
.unwrap()
.extract::<String>()
.unwrap(),
"custom"
);
} else {
assert!(optional_params.is_none());
}
});
}

View file

@ -0,0 +1,27 @@
use std::fmt;
use litellm_secrets::Error;
use pyo3::{exceptions::PyBaseException, prelude::*};
#[derive(thiserror::Error)]
#[error("Python secret manager failed")]
struct PythonSecretError(Py<PyBaseException>);
impl fmt::Debug for PythonSecretError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("PythonSecretError")
}
}
pub(super) fn external_error(py: Python<'_>, error: PyErr) -> Error {
Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py))))
}
pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option<PyErr> {
let Error::ExternalManager(source) = error else {
return None;
};
source
.downcast_ref::<PythonSecretError>()
.map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any()))
}

View file

@ -1,3 +1,6 @@
pub(crate) mod callback;
pub(crate) mod config;
mod error;
pub(crate) mod resolved;
pub(crate) use error::python_error;

View file

@ -1,158 +0,0 @@
use std::sync::Arc;
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use std::{num::NonZero, thread::available_parallelism};
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
use litellm_host_python::release_gil;
use litellm_host_python::run_async;
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
types::PyAny,
};
use tokio::sync::Semaphore;
use crate::errors::RustBridgeDeclined;
/// Counts the input tokens of a raw request body off the Python event loop with
/// the GIL released. Python owns which requests get here and what to do with
/// the count. At most one encode per core runs at a time; the rest wait in the
/// async task, where a cancelled Python awaiter drops them before any blocking
/// work is scheduled.
#[pyclass(frozen)]
pub(crate) struct TokenCounter {
inner: Arc<CoreTokenCounter>,
encode_slots: Arc<Semaphore>,
}
#[pymethods]
impl TokenCounter {
#[new]
fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json))
}
#[cfg(all(not(feature = "fast"), feature = "huggingface"))]
{
Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
}
#[cfg(not(any(feature = "fast", feature = "huggingface")))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast or huggingface feature",
))
}
}
#[staticmethod]
fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
}
#[cfg(not(feature = "fast"))]
{
let _ = (py, rank_file);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast feature",
))
}
}
#[staticmethod]
fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
#[cfg(feature = "fast")]
{
Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
}
#[cfg(not(feature = "fast"))]
{
let _ = (py, rank_file);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the fast feature",
))
}
}
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
Self::load(py, || CoreTokenCounter::from_tiktoken(encoding))
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
let counter = Arc::clone(&self.inner);
let encode_slots = Arc::clone(&self.encode_slots);
let body = body.to_vec();
run_async(
py,
async move {
let _slot = encode_slots
.acquire_owned()
.await
.map_err(|error| Error::Task(error.to_string()))?;
tokio::task::spawn_blocking(move || count_body(&counter, &body))
.await
.map_err(|error| Error::Task(error.to_string()))?
},
token_count_error_to_pyerr,
)
}
}
impl TokenCounter {
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
fn load(
py: Python<'_>,
load: impl FnOnce() -> Result<CoreTokenCounter, Error> + Send,
) -> PyResult<Self> {
let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?;
Ok(Self {
inner: Arc::new(inner),
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
})
}
}
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
fn encode_parallelism() -> usize {
available_parallelism().map_or(1, NonZero::get)
}
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
let request = CountableRequest::parse(body)?;
counter.count_request(&request)
}
fn token_count_error_to_pyerr(error: Error) -> PyErr {
let message = error.to_string();
match error {
Error::Load(_)
| Error::Ranks(_)
| Error::UnicodeClasses
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
Error::RequestParse(_)
| Error::MissingInput
| Error::FloatText
| Error::ContentBlock
| Error::ArrayItems
| Error::JsonSerialization(_)
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
}
}

View file

@ -0,0 +1,713 @@
//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging
//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and
//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps.
use std::borrow::Cow;
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
use std::collections::HashMap;
use std::sync::Arc;
#[cfg(feature = "fast")]
use std::sync::OnceLock;
use litellm_host_python::{enter_native, release_gil};
#[cfg(feature = "fast")]
use litellm_token_counter::fast::{FastCounter, FastTokenizer};
use litellm_token_counter::{Error, TextCodec};
use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString};
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
use pyo3::exceptions::PyValueError;
#[cfg(feature = "huggingface")]
use pyo3::{exceptions::PyIOError, types::PyDict};
#[cfg(feature = "tiktoken")]
use pyo3::{
exceptions::{PyKeyError, PyRuntimeError},
types::PyBytes,
};
#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))]
use crate::errors::RustBridgeDeclined;
use crate::routes::token_counter::token_count_error_to_pyerr;
#[cfg(feature = "huggingface")]
use litellm_token_counter::huggingface::{
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy,
TruncationDirection, encoding_from_json, encoding_to_json,
};
#[cfg(feature = "tiktoken")]
use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary};
#[cfg(feature = "tiktoken")]
pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<TiktokenTokenizer> {
enter_native()?;
let resource: std::path::PathBuf =
PyModule::import(py, "litellm.litellm_core_utils.tokenizers")?
.getattr("__file__")?
.extract()?;
release_gil(py, || {
TiktokenTokenizer::from_cached_ranks(encoding, |file| {
std::fs::read_to_string(resource.with_file_name(file))
})
})
.map_err(|error| token_count_error_to_pyerr(error.into()))
}
pub(crate) enum Codec {
#[cfg(feature = "tiktoken")]
Tiktoken(TiktokenTokenizer),
#[cfg(feature = "huggingface")]
HuggingFace(HuggingFaceTokenizer),
}
impl Codec {
pub(crate) fn codec(&self) -> &dyn TextCodec {
match *self {
#[cfg(feature = "tiktoken")]
Self::Tiktoken(ref tokenizer) => tokenizer,
#[cfg(feature = "huggingface")]
Self::HuggingFace(ref tokenizer) => tokenizer,
}
}
#[cfg(feature = "fast")]
fn fast_counter(&self) -> Option<FastTokenizer> {
match *self {
#[cfg(feature = "tiktoken")]
Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(),
#[cfg(feature = "huggingface")]
Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(),
}
}
}
/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse,
/// and the opt-in count-only counter is derived from it once, on first use.
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
pub(crate) struct Tokenizer {
inner: Arc<Codec>,
#[cfg(feature = "fast")]
fast: OnceLock<Option<Arc<FastTokenizer>>>,
}
#[pymethods]
impl Tokenizer {
#[staticmethod]
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
#[cfg(feature = "tiktoken")]
{
let tokenizer = load_tiktoken(py, encoding)?;
Ok(Self::new(Codec::Tiktoken(tokenizer)))
}
#[cfg(not(feature = "tiktoken"))]
{
let _ = (py, encoding);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the tiktoken feature",
))
}
}
#[staticmethod]
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
enter_native()?;
let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json))
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
Ok(Self::new(Codec::HuggingFace(tokenizer)))
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, tokenizer_json);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
#[staticmethod]
#[pyo3(signature = (identifier, revision = "main", token = None))]
fn from_pretrained(
py: Python<'_>,
identifier: &str,
revision: &str,
token: Option<&str>,
) -> PyResult<Self> {
#[cfg(feature = "huggingface")]
{
enter_native()?;
let kwargs = PyDict::new(py);
kwargs.set_item("repo_id", identifier)?;
kwargs.set_item("filename", "tokenizer.json")?;
kwargs.set_item("revision", revision)?;
kwargs.set_item("token", token)?;
let path: String = PyModule::import(py, "huggingface_hub")?
.getattr("hf_hub_download")?
.call((), Some(&kwargs))?
.extract()?;
let json =
release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?;
Self::from_json(py, &json)
}
#[cfg(not(feature = "huggingface"))]
{
let _ = (py, identifier, revision, token);
Err(RustBridgeDeclined::new_err(
"tokenizer backend requires the huggingface feature",
))
}
}
fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<Vec<u32>> {
enter_native()?;
let text = self.text(text)?;
release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (ids, skip_special_tokens = true))]
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
enter_native()?;
release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens))
.map_err(token_count_error_to_pyerr)
}
#[pyo3(signature = (text, fast = false))]
fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult<usize> {
enter_native()?;
let text = self.text(text)?;
let counter = self.counter(py, fast);
release_gil(py, || {
litellm_token_counter::Tokenizer::count_tokens(&counter, &text)
})
.map_err(token_count_error_to_pyerr)
}
#[getter]
fn name(&self) -> &str {
self.inner.codec().name()
}
// ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------
#[cfg(feature = "tiktoken")]
fn encode_special(
&self,
py: Python<'_>,
text: &Bound<'_, PyString>,
allowed: Vec<String>,
) -> PyResult<Vec<u32>> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let text = self.text(text)?;
release_gil(py, || tokenizer.encode_special(&text, &allowed))
.map_err(PyRuntimeError::new_err)
}
/// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`.
#[cfg(feature = "tiktoken")]
fn encode_with_unstable(
&self,
py: Python<'_>,
text: &Bound<'_, PyString>,
allowed: Vec<String>,
) -> PyResult<(Vec<u32>, Vec<Vec<u32>>)> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let text = self.text(text)?;
Ok(release_gil(py, || {
tokenizer.encode_with_unstable(&text, &allowed)
}))
}
/// The special tokens by text: tiktoken's `_special_tokens`.
#[cfg(feature = "tiktoken")]
fn special_tokens(&self) -> PyResult<HashMap<String, u32>> {
Ok(self
.vocabulary()?
.special_tokens()
.map(|(token, rank)| (token.to_owned(), rank))
.collect())
}
#[cfg(feature = "tiktoken")]
fn max_token_value(&self) -> PyResult<u32> {
Ok(self.vocabulary()?.max_token_value())
}
#[cfg(feature = "tiktoken")]
fn is_special_token(&self, token: u32) -> PyResult<bool> {
Ok(self.vocabulary()?.is_special_token(token))
}
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
#[cfg(feature = "tiktoken")]
fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyBytes>>> {
let vocabulary = self.vocabulary()?;
let values = release_gil(py, || vocabulary.token_byte_values());
Ok(values.iter().map(|value| PyBytes::new(py, value)).collect())
}
/// The token of one whole piece; `KeyError` when it is not in the vocabulary.
#[cfg(feature = "tiktoken")]
fn encode_single_token(&self, py: Python<'_>, piece: Vec<u8>) -> PyResult<u32> {
self.vocabulary()?
.encode_single_token(&piece)
.ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind()))
}
#[cfg(feature = "tiktoken")]
fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec<u32>) -> PyResult<Bound<'py, PyBytes>> {
enter_native()?;
let tokenizer = self.tiktoken()?;
let bytes =
release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?;
Ok(PyBytes::new(py, &bytes))
}
// ---- Hugging Face: the `tokenizers.Tokenizer` surface -----------------------------------
#[cfg(feature = "huggingface")]
#[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_huggingface(
&self,
py: Python<'_>,
sequence: Sequence,
pair: Option<Sequence>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<HuggingFaceEncoding> {
enter_native()?;
let tokenizer = self.huggingface()?;
let sequence = sequence.input(is_pretokenized)?;
let input = match pair {
Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?),
None => EncodeInput::Single(sequence),
};
release_gil(py, || {
tokenizer.encode_result(input, add_special_tokens, fast)
})
.map(|inner| HuggingFaceEncoding { inner })
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))]
fn encode_batch_huggingface(
&self,
py: Python<'_>,
inputs: Vec<(Sequence, Option<Sequence>)>,
is_pretokenized: bool,
add_special_tokens: bool,
fast: bool,
) -> PyResult<Vec<HuggingFaceEncoding>> {
enter_native()?;
let tokenizer = self.huggingface()?;
let inputs = inputs
.into_iter()
.map(|(sequence, pair)| {
let sequence = sequence.input(is_pretokenized)?;
match pair {
Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)),
None => Ok(EncodeInput::Single(sequence)),
}
})
.collect::<PyResult<Vec<_>>>()?;
release_gil(py, || {
tokenizer.encode_batch_result(inputs, add_special_tokens, fast)
})
.map(|encodings| {
encodings
.into_iter()
.map(|inner| HuggingFaceEncoding { inner })
.collect()
})
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (pretty = false))]
fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult<String> {
enter_native()?;
let tokenizer = self.huggingface()?;
release_gil(py, || tokenizer.to_json(pretty))
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
}
#[cfg(feature = "huggingface")]
fn token_to_id(&self, token: &str) -> PyResult<Option<u32>> {
Ok(self.huggingface()?.token_to_id(token))
}
#[cfg(feature = "huggingface")]
fn id_to_token(&self, id: u32) -> PyResult<Option<String>> {
Ok(self.huggingface()?.id_to_token(id))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (with_added_tokens = true))]
fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult<HashMap<String, u32>> {
let tokenizer = self.huggingface()?;
Ok(release_gil(py, || tokenizer.vocab(with_added_tokens)))
}
#[cfg(feature = "huggingface")]
#[pyo3(signature = (with_added_tokens = true))]
fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult<usize> {
Ok(self.huggingface()?.vocab_size(with_added_tokens))
}
/// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized,
/// special))`, for Python to rebuild as `tokenizers.AddedToken`.
#[cfg(feature = "huggingface")]
fn added_tokens_decoder(&self) -> PyResult<Vec<(u32, AddedTokenFields)>> {
Ok(self
.huggingface()?
.added_tokens_decoder()
.into_iter()
.map(|(id, token)| {
(
id,
(
token.content,
token.single_word,
token.lstrip,
token.rstrip,
token.normalized,
token.special,
),
)
})
.collect())
}
/// The padding parameters as `tokenizers.Tokenizer.padding` reports them.
#[cfg(feature = "huggingface")]
fn padding<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
let Some(params) = self.huggingface()?.padding() else {
return Ok(None);
};
let padding = PyDict::new(py);
padding.set_item(
"length",
match params.strategy {
PaddingStrategy::BatchLongest => None,
PaddingStrategy::Fixed(length) => Some(length),
},
)?;
padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?;
padding.set_item("pad_id", params.pad_id)?;
padding.set_item("pad_type_id", params.pad_type_id)?;
padding.set_item("pad_token", &params.pad_token)?;
padding.set_item("direction", params.direction.as_ref())?;
Ok(Some(padding))
}
/// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them.
#[cfg(feature = "huggingface")]
fn truncation<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
let Some(params) = self.huggingface()?.truncation() else {
return Ok(None);
};
let truncation = PyDict::new(py);
truncation.set_item("max_length", params.max_length)?;
truncation.set_item("stride", params.stride)?;
truncation.set_item("strategy", params.strategy.as_ref())?;
truncation.set_item("direction", params.direction.as_ref())?;
Ok(Some(truncation))
}
#[cfg(feature = "huggingface")]
fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult<usize> {
Ok(self.huggingface()?.num_special_tokens_to_add(is_pair))
}
#[cfg(feature = "huggingface")]
fn encode_special_tokens(&self) -> PyResult<bool> {
Ok(self.huggingface()?.encode_special_tokens())
}
}
#[cfg(feature = "huggingface")]
type AddedTokenFields = (String, bool, bool, bool, bool, bool);
impl Tokenizer {
fn new(inner: Codec) -> Self {
Self {
inner: Arc::new(inner),
#[cfg(feature = "fast")]
fast: OnceLock::new(),
}
}
pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter {
#[cfg(feature = "fast")]
if fast {
let counter = self.fast.get().unwrap_or_else(|| {
release_gil(py, || {
self.fast
.get_or_init(|| self.inner.fast_counter().map(Arc::new))
})
});
if let Some(counter) = counter {
return SharedCounter::Fast(Arc::clone(counter));
}
}
#[cfg(not(feature = "fast"))]
let _ = (py, fast);
SharedCounter::Codec(Arc::clone(&self.inner))
}
/// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode`
/// does; `tokenizers` rejects them, so that backend keeps the encode error.
fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult<Cow<'a, str>> {
match text.to_cow() {
Ok(text) => Ok(text),
Err(error) => match *self.inner {
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) if error.is_instance_of::<PyUnicodeEncodeError>(text.py()) => {
text.call_method1("encode", ("utf-16", "surrogatepass"))?
.call_method1("decode", ("utf-16", "replace"))?
.extract::<String>()
.map(Cow::Owned)
}
_ => Err(error),
},
}
}
#[cfg(feature = "tiktoken")]
fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> {
match *self.inner {
Codec::Tiktoken(ref tokenizer) => Ok(tokenizer),
#[cfg(feature = "huggingface")]
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
}
}
#[cfg(feature = "tiktoken")]
fn vocabulary(&self) -> PyResult<&Vocabulary> {
self.tiktoken()?.vocabulary().ok_or_else(|| {
PyRuntimeError::new_err("this encoding was built without its vocabulary")
})
}
#[cfg(feature = "huggingface")]
fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> {
match *self.inner {
Codec::HuggingFace(ref tokenizer) => Ok(tokenizer),
#[cfg(feature = "tiktoken")]
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
}
}
}
pub(crate) enum SharedCounter {
Codec(Arc<Codec>),
#[cfg(feature = "fast")]
Fast(Arc<FastTokenizer>),
}
impl litellm_token_counter::Tokenizer for SharedCounter {
fn count_tokens(&self, text: &str) -> Result<usize, Error> {
match self {
Self::Codec(codec) => codec.codec().count_tokens(text),
#[cfg(feature = "fast")]
Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from),
}
}
}
#[cfg(feature = "huggingface")]
#[derive(FromPyObject)]
pub(crate) enum Sequence {
Text(String),
Words(Vec<String>),
}
#[cfg(feature = "huggingface")]
impl Sequence {
fn input(self, is_pretokenized: bool) -> PyResult<InputSequence<'static>> {
match (self, is_pretokenized) {
(Self::Text(text), false) => Ok(text.into()),
(Self::Words(words), true) => Ok(words.into()),
_ => Err(pyo3::exceptions::PyTypeError::new_err(
"input must match is_pretokenized",
)),
}
}
}
#[cfg(feature = "huggingface")]
fn direction<T>(value: &str, left: T, right: T, what: &str) -> PyResult<T> {
match value {
"left" => Ok(left),
"right" => Ok(right),
other => Err(PyValueError::new_err(format!(
"invalid {what} direction {other:?}: expected 'left' or 'right'"
))),
}
}
/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id`
/// change it in place.
#[cfg(feature = "huggingface")]
#[pyclass(module = "litellm.rust_bridge._native")]
pub(crate) struct HuggingFaceEncoding {
inner: Encoding,
}
#[cfg(feature = "huggingface")]
#[pymethods]
impl HuggingFaceEncoding {
#[new]
#[pyo3(signature = (json = None))]
fn new(json: Option<&str>) -> PyResult<Self> {
let inner = match json {
Some(json) => encoding_from_json(json)
.map_err(|error| PyValueError::new_err(error.to_string()))?,
None => Encoding::default(),
};
Ok(Self { inner })
}
#[staticmethod]
#[pyo3(signature = (encodings, growing_offsets = true))]
fn merge(encodings: Vec<PyRef<'_, Self>>, growing_offsets: bool) -> Self {
Self {
inner: Encoding::merge(
encodings.iter().map(|encoding| encoding.inner.clone()),
growing_offsets,
),
}
}
fn __reduce__<'py>(
&self,
py: Python<'py>,
) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
let json = encoding_to_json(&self.inner)
.map_err(|error| PyValueError::new_err(error.to_string()))?;
Ok((py.get_type::<Self>(), (json,)))
}
fn __repr__(&self) -> String {
format!(
"Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \
attention_mask, special_tokens_mask, overflowing])",
self.inner.len()
)
}
fn __len__(&self) -> usize {
self.inner.len()
}
#[getter]
fn ids(&self) -> Vec<u32> {
self.inner.get_ids().to_vec()
}
#[getter]
fn tokens(&self) -> Vec<String> {
self.inner.get_tokens().to_vec()
}
#[getter]
fn offsets(&self) -> Vec<(usize, usize)> {
self.inner.get_offsets().to_vec()
}
#[getter]
fn type_ids(&self) -> Vec<u32> {
self.inner.get_type_ids().to_vec()
}
#[getter]
fn attention_mask(&self) -> Vec<u32> {
self.inner.get_attention_mask().to_vec()
}
#[getter]
fn special_tokens_mask(&self) -> Vec<u32> {
self.inner.get_special_tokens_mask().to_vec()
}
#[getter]
fn word_ids(&self) -> Vec<Option<u32>> {
self.inner.get_word_ids().to_vec()
}
#[getter]
fn sequence_ids(&self) -> Vec<Option<usize>> {
self.inner.get_sequence_ids()
}
#[getter]
fn overflowing(&self) -> Vec<Self> {
self.inner
.get_overflowing()
.iter()
.cloned()
.map(|inner| Self { inner })
.collect()
}
#[getter]
fn n_sequences(&self) -> usize {
self.inner.n_sequences()
}
#[pyo3(signature = (word_index, sequence_index = 0))]
fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
self.inner.word_to_tokens(word_index, sequence_index)
}
#[pyo3(signature = (word_index, sequence_index = 0))]
fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
self.inner.word_to_chars(word_index, sequence_index)
}
fn token_to_sequence(&self, token_index: usize) -> Option<usize> {
self.inner.token_to_sequence(token_index)
}
fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> {
self.inner
.token_to_chars(token_index)
.map(|(_, offsets)| offsets)
}
fn token_to_word(&self, token_index: usize) -> Option<u32> {
self.inner.token_to_word(token_index).map(|(_, word)| word)
}
#[pyo3(signature = (char_pos, sequence_index = 0))]
fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option<usize> {
self.inner.char_to_token(char_pos, sequence_index)
}
#[pyo3(signature = (char_pos, sequence_index = 0))]
fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option<u32> {
self.inner.char_to_word(char_pos, sequence_index)
}
fn set_sequence_id(&mut self, sequence_id: usize) {
self.inner.set_sequence_id(sequence_id);
}
#[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))]
fn pad(
&mut self,
length: usize,
direction: &str,
pad_id: u32,
pad_type_id: u32,
pad_token: &str,
) -> PyResult<()> {
let direction = self::direction(
direction,
PaddingDirection::Left,
PaddingDirection::Right,
"padding",
)?;
self.inner
.pad(length, pad_id, pad_type_id, pad_token, direction);
Ok(())
}
#[pyo3(signature = (max_length, stride = 0, direction = "right"))]
fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> {
let direction = self::direction(
direction,
TruncationDirection::Left,
TruncationDirection::Right,
"truncation",
)?;
self.inner.truncate(max_length, stride, direction);
Ok(())
}
}

View file

@ -6,6 +6,10 @@ pub enum Error {
Auth(#[from] #[redact] litellm_auth_aws::Error),
#[error("AWS region is not configured")]
MissingRegion,
#[error("AWS Secrets Manager received a non-AWS operation context")]
InvalidOperationContext,
#[error("AWS Secrets Manager was constructed without context-aware configuration")]
OperationContextUnavailable,
#[error("KMS response has no plaintext")]
MissingPlaintext,
#[error("AWS request timed out")]

View file

@ -16,7 +16,8 @@ use litellm_auth_aws::constants::{
};
use litellm_core_utils::settings::Lookup;
use litellm_secrets_types::{
BaseSecretManager, KeyManagementSettings, Secret, SecretValue, async_rotate_secret,
AwsOperationContext, BaseSecretManager, KeyManagementSettings, Secret, SecretOperationContext,
SecretValue, SecretWriteContext, async_rotate_secret,
};
use serde_json::Value;
@ -25,9 +26,17 @@ use crate::{Error, auth};
#[derive(Clone)]
pub struct AwsSecretsManagerV2 {
client: Client,
context_client_factory: Option<Box<ContextClientFactory>>,
write_settings: AwsSecretWriteSettings,
}
#[derive(Clone)]
struct ContextClientFactory {
settings: KeyManagementSettings,
environment: Arc<dyn Lookup + Send + Sync>,
endpoint_url: Option<String>,
}
#[derive(Clone, Debug, Default)]
pub struct AwsSecretWriteSettings {
pub kms_key_id: Option<String>,
@ -55,6 +64,19 @@ impl AwsSecretsManagerV2 {
pub fn new(client: Client, write_settings: AwsSecretWriteSettings) -> Self {
Self {
client,
context_client_factory: None,
write_settings,
}
}
fn with_context_client_factory(
client: Client,
write_settings: AwsSecretWriteSettings,
context_client_factory: ContextClientFactory,
) -> Self {
Self {
client,
context_client_factory: Some(Box::new(context_client_factory)),
write_settings,
}
}
@ -67,19 +89,18 @@ impl AwsSecretsManagerV2 {
if use_aws_secret_manager != Some(true) {
return Ok(None);
}
let builder = aws_sdk_secretsmanager::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(auth::region(&settings, environment.as_ref())?))
.credentials_provider(auth::Credentials::new(&settings, environment.clone()));
let config = match environment.get(AWS_BEDROCK_RUNTIME_ENDPOINT) {
Some(url) => builder
.endpoint_url(url.replace("bedrock-runtime", "secretsmanager"))
.build(),
None => builder.build(),
let context_client_factory = ContextClientFactory {
settings: settings.clone(),
environment: environment.clone(),
endpoint_url: environment
.get(AWS_BEDROCK_RUNTIME_ENDPOINT)
.map(|url| url.replace("bedrock-runtime", "secretsmanager")),
};
Ok(Some(Self::new(
Client::from_conf(config),
let client = context_client_factory.client(&AwsOperationContext::default())?;
Ok(Some(Self::with_context_client_factory(
client,
(&settings).into(),
context_client_factory,
)))
}
@ -118,7 +139,14 @@ impl AwsSecretsManagerV2 {
}
pub async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
match self.client.get_secret_value().secret_id(name).send().await {
Self::async_read_secret_with_client(&self.client, name).await
}
async fn async_read_secret_with_client(
client: &Client,
name: &str,
) -> Result<Option<SecretValue>, Error> {
match client.get_secret_value().secret_id(name).send().await {
Ok(response) => response
.secret_string
.map(SecretValue::new)
@ -149,8 +177,19 @@ impl AwsSecretsManagerV2 {
value: &SecretValue,
description: Option<&str>,
) -> Result<CreateSecretOutput, Error> {
let response = self
.client
self.async_write_secret_with_client_and_tags(&self.client, name, value, description, None)
.await
}
async fn async_write_secret_with_client_and_tags(
&self,
client: &Client,
name: &str,
value: &SecretValue,
description: Option<&str>,
tags: Option<&BTreeMap<String, String>>,
) -> Result<CreateSecretOutput, Error> {
let response = client
.create_secret()
.name(name)
.secret_string(value.expose())
@ -161,7 +200,7 @@ impl AwsSecretsManagerV2 {
.clone()
.filter(|v| !v.is_empty()),
)
.set_tags(self.write_settings.tags.as_ref().map(|tags| {
.set_tags(tags.or(self.write_settings.tags.as_ref()).map(|tags| {
tags.iter()
.map(|(key, value)| Tag::builder().key(key).value(value).build())
.collect()
@ -171,7 +210,10 @@ impl AwsSecretsManagerV2 {
.map_err(|error| Error::Create(Box::new(error)))?;
if let Some(regions) = &self.write_settings.replica_regions
&& !regions.is_empty()
&& self.async_replicate_secret(name, regions).await.is_err()
&& self
.async_replicate_secret_with_client(client, name, regions)
.await
.is_err()
{
tracing::warn!("secret created but replication failed");
}
@ -182,11 +224,21 @@ impl AwsSecretsManagerV2 {
&self,
name: &str,
regions: &[String],
) -> Result<Option<ReplicateSecretToRegionsOutput>, Error> {
self.async_replicate_secret_with_client(&self.client, name, regions)
.await
}
async fn async_replicate_secret_with_client(
&self,
client: &Client,
name: &str,
regions: &[String],
) -> Result<Option<ReplicateSecretToRegionsOutput>, Error> {
if regions.is_empty() {
return Ok(None);
}
self.client
client
.replicate_secret_to_regions()
.secret_id(name)
.set_add_replica_regions(Some(
@ -206,7 +258,17 @@ impl AwsSecretsManagerV2 {
name: &str,
value: &SecretValue,
) -> Result<PutSecretValueOutput, Error> {
self.client
self.async_put_secret_value_with_client(&self.client, name, value)
.await
}
async fn async_put_secret_value_with_client(
&self,
client: &Client,
name: &str,
value: &SecretValue,
) -> Result<PutSecretValueOutput, Error> {
client
.put_secret_value()
.secret_id(name)
.secret_string(value.expose())
@ -218,12 +280,22 @@ impl AwsSecretsManagerV2 {
pub async fn async_delete_secret(
&self,
name: &str,
recovery_window_in_days: i64,
recovery_window_in_days: Option<u32>,
) -> Result<DeleteSecretOutput, Error> {
self.client
self.async_delete_secret_with_client(&self.client, name, recovery_window_in_days)
.await
}
async fn async_delete_secret_with_client(
&self,
client: &Client,
name: &str,
recovery_window_in_days: Option<u32>,
) -> Result<DeleteSecretOutput, Error> {
client
.delete_secret()
.secret_id(name)
.recovery_window_in_days(recovery_window_in_days)
.set_recovery_window_in_days(recovery_window_in_days.map(i64::from))
.send()
.await
.map_err(|error| Error::Delete(Box::new(error)))
@ -234,17 +306,105 @@ impl AwsSecretsManagerV2 {
current_name: &str,
new_name: &str,
value: &SecretValue,
) -> Result<RotationResponse, Error> {
self.async_rotate_secret_with_context(
current_name,
new_name,
value,
&SecretOperationContext::default(),
)
.await
}
pub async fn async_rotate_secret_with_context(
&self,
current_name: &str,
new_name: &str,
value: &SecretValue,
context: &SecretOperationContext,
) -> Result<RotationResponse, Error> {
if current_name == new_name {
let client = self.client_for_context(context)?;
return self
.async_put_secret_value(current_name, value)
.async_put_secret_value_with_client(&client, current_name, value)
.await
.map(RotationResponse::Updated);
}
async_rotate_secret(self, current_name, new_name, value)
async_rotate_secret(self, current_name, new_name, value, context)
.await
.map(RotationResponse::Created)
}
fn client_for_context(&self, context: &SecretOperationContext) -> Result<Client, Error> {
match context {
SecretOperationContext::Default => Ok(self.client.clone()),
SecretOperationContext::Aws(context) if context == &AwsOperationContext::default() => {
Ok(self.client.clone())
}
SecretOperationContext::Aws(context) => self
.context_client_factory
.as_ref()
.ok_or(Error::OperationContextUnavailable)?
.client(context),
_ => Err(Error::InvalidOperationContext),
}
}
}
impl ContextClientFactory {
fn client(&self, context: &AwsOperationContext) -> Result<Client, Error> {
let settings = KeyManagementSettings {
aws_region_name: context
.region_name
.clone()
.or_else(|| self.settings.aws_region_name.clone()),
aws_role_name: context
.role_name
.clone()
.or_else(|| self.settings.aws_role_name.clone()),
aws_session_name: context
.session_name
.clone()
.or_else(|| self.settings.aws_session_name.clone()),
aws_external_id: context
.external_id
.clone()
.or_else(|| self.settings.aws_external_id.clone()),
aws_profile_name: context
.profile_name
.clone()
.or_else(|| self.settings.aws_profile_name.clone()),
aws_web_identity_token: context
.web_identity_token
.clone()
.or_else(|| self.settings.aws_web_identity_token.clone()),
aws_sts_endpoint: context
.sts_endpoint
.clone()
.or_else(|| self.settings.aws_sts_endpoint.clone()),
..self.settings.clone()
};
let builder = aws_sdk_secretsmanager::Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(auth::region(
&settings,
self.environment.as_ref(),
)?))
.credentials_provider(auth::Credentials::new(&settings, self.environment.clone()));
let builder = match context.timeout {
Some(timeout) => builder.timeout_config(
aws_sdk_secretsmanager::config::timeout::TimeoutConfig::builder()
.operation_timeout(timeout)
.build(),
),
None => builder,
};
let config = match &self.endpoint_url {
Some(endpoint_url) => builder.endpoint_url(endpoint_url.clone()).build(),
None => builder.build(),
};
Ok(Client::from_conf(config))
}
}
impl BaseSecretManager for AwsSecretsManagerV2 {
@ -252,25 +412,40 @@ impl BaseSecretManager for AwsSecretsManagerV2 {
type WriteResponse = CreateSecretOutput;
type DeleteResponse = DeleteSecretOutput;
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
self.async_read_secret(name).await
async fn async_read_secret(
&self,
name: &str,
context: &SecretOperationContext,
) -> Result<Option<SecretValue>, Error> {
let client = self.client_for_context(context)?;
Self::async_read_secret_with_client(&client, name).await
}
async fn async_write_secret(
&self,
name: &str,
value: &SecretValue,
description: Option<&str>,
context: &SecretWriteContext,
) -> Result<CreateSecretOutput, Error> {
self.async_write_secret(name, value, description).await
let client = self.client_for_context(&context.operation)?;
self.async_write_secret_with_client_and_tags(
&client,
name,
value,
context.description.as_deref(),
(!context.tags.is_empty()).then_some(&context.tags),
)
.await
}
async fn async_delete_secret(
&self,
name: &str,
recovery_window_in_days: i64,
recovery_window_in_days: Option<u32>,
context: &SecretOperationContext,
) -> Result<DeleteSecretOutput, Error> {
self.async_delete_secret(name, recovery_window_in_days)
let client = self.client_for_context(context)?;
self.async_delete_secret_with_client(&client, name, recovery_window_in_days)
.await
}
}

View file

@ -3,12 +3,15 @@ use aws_sdk_kms::{
config::{BehaviorVersion, Credentials, Region, retry::RetryConfig},
};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_secrets_aws::AwsKms;
use litellm_secrets_aws::{AwsKms, Error, load_aws_kms};
use litellm_secrets_types::KeyManagementSettings;
use rstest::rstest;
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{body_json, header},
};
#[rstest]
#[tokio::test]
async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() {
let server = MockServer::start().await;
@ -40,20 +43,56 @@ async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() {
);
}
#[test]
fn disabled_kms_loader_does_not_require_environment_configuration() {
use litellm_secrets_aws::load_aws_kms;
use litellm_secrets_types::KeyManagementSettings;
#[rstest]
#[case::unset(None)]
#[case::disabled(Some(false))]
fn disabled_kms_loader_does_not_require_environment_configuration(#[case] enabled: Option<bool>) {
use std::sync::Arc;
for enabled in [None, Some(false)] {
assert!(
load_aws_kms(
enabled,
&KeyManagementSettings::default(),
Arc::new(|_: &str| None)
)
.unwrap()
.is_none()
);
}
assert!(
load_aws_kms(
enabled,
&KeyManagementSettings::default(),
Arc::new(|_: &str| None)
)
.unwrap()
.is_none()
);
}
#[rstest]
#[case::settings(Some("configured-region"), None)]
#[case::environment(None, Some("environment-region"))]
fn enabled_kms_loader_accepts_either_region_source(
#[case] configured_region: Option<&'static str>,
#[case] environment_region: Option<&'static str>,
) {
use std::sync::Arc;
let settings = KeyManagementSettings {
aws_region_name: configured_region.map(str::to_owned),
..KeyManagementSettings::default()
};
let environment = Arc::new(move |name: &str| {
(name == "AWS_REGION_NAME")
.then(|| environment_region.map(str::to_owned))
.flatten()
});
assert!(
load_aws_kms(Some(true), &settings, environment)
.unwrap()
.is_some()
);
}
#[rstest]
fn enabled_kms_loader_rejects_missing_region() {
use std::sync::Arc;
assert!(matches!(
load_aws_kms(
Some(true),
&KeyManagementSettings::default(),
Arc::new(|_: &str| None),
),
Err(Error::MissingRegion)
));
}

View file

@ -1,11 +1,21 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use std::{
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
time::Duration,
};
use aws_sdk_secretsmanager::{
Client,
config::{BehaviorVersion, Credentials, Region, retry::RetryConfig},
};
use litellm_secrets_aws::{AwsSecretsManagerV2, Error, RotationResponse};
use litellm_secrets_types::{KeyManagementSettings, SecretValue};
use litellm_secrets_types::{
AwsOperationContext, BaseSecretManager, KeyManagementSettings, SecretOperationContext,
SecretValue, SecretWriteContext,
};
use rstest::{fixture, rstest};
use serde_json::json;
use wiremock::{
Mock, MockServer, ResponseTemplate,
@ -25,12 +35,39 @@ fn manager(server: &MockServer, settings: KeyManagementSettings) -> AwsSecretsMa
AwsSecretsManagerV2::new(client, (&settings).into())
}
#[rstest::rstest]
fn loaded_manager(server: &MockServer) -> AwsSecretsManagerV2 {
let endpoint_url = server.uri();
let environment: Arc<dyn litellm_core_utils::settings::Lookup + Send + Sync> =
Arc::new(move |name: &str| match name {
"AWS_BEDROCK_RUNTIME_ENDPOINT" => Some(endpoint_url.clone()),
"AWS_ACCESS_KEY_ID" => Some("test".into()),
"AWS_SECRET_ACCESS_KEY" => Some("test".into()),
_ => None,
});
AwsSecretsManagerV2::load_aws_secret_manager(
Some(true),
KeyManagementSettings {
aws_region_name: Some("us-east-1".into()),
..Default::default()
},
environment,
)
.unwrap()
.unwrap()
}
#[fixture]
fn default_settings() -> KeyManagementSettings {
KeyManagementSettings::default()
}
#[rstest]
#[case::string_value("KEY", Some("value"))]
#[case::missing_value("missing", None)]
#[case::non_string_value("BOOL", None)]
#[tokio::test]
async fn primary_lookup_preserves_read_semantics(
default_settings: KeyManagementSettings,
#[case] name: &str,
#[case] expected: Option<&str>,
) {
@ -45,7 +82,7 @@ async fn primary_lookup_preserves_read_semantics(
.expect(1)
.mount(&server)
.await;
let manager = manager(&server, KeyManagementSettings::default());
let manager = manager(&server, default_settings);
assert_eq!(
manager
.read_secret_for_resolver(name, Some("primary"), &|_: &str| None)
@ -57,16 +94,19 @@ async fn primary_lookup_preserves_read_semantics(
);
}
#[rstest::rstest]
#[rstest]
#[case::access_key("AWS_ACCESS_KEY_ID")]
#[case::secret_access_key("AWS_SECRET_ACCESS_KEY")]
#[case::region_name("AWS_REGION_NAME")]
#[case::region("AWS_REGION")]
#[case::bedrock_endpoint("AWS_BEDROCK_RUNTIME_ENDPOINT")]
#[tokio::test]
async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) {
async fn bootstrap_keys_bypass_primary_lookup(
default_settings: KeyManagementSettings,
#[case] name: &str,
) {
let server = MockServer::start().await;
let manager = manager(&server, KeyManagementSettings::default());
let manager = manager(&server, default_settings);
assert_eq!(
manager
.read_secret_for_resolver(name, Some("primary"), &|_: &str| Some("bootstrap".into()))
@ -79,8 +119,11 @@ async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) {
);
}
#[rstest]
#[tokio::test]
async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() {
async fn failed_read_returns_none_but_invalid_primary_json_is_an_error(
default_settings: KeyManagementSettings,
) {
let server = MockServer::start().await;
Mock::given(body_partial_json(json!({"SecretId":"missing"})))
.respond_with(
@ -92,7 +135,11 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() {
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"SecretString":"not-json"})))
.mount(&server)
.await;
let manager = manager(&server, KeyManagementSettings::default());
Mock::given(body_partial_json(json!({"SecretId":"no-string"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name":"no-string"})))
.mount(&server)
.await;
let manager = manager(&server, default_settings);
assert!(
manager
.async_read_secret("missing")
@ -106,10 +153,17 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() {
.await,
Err(Error::PrimarySecret)
));
assert!(matches!(
manager.async_read_secret("no-string").await,
Err(Error::MissingString)
));
}
#[rstest]
#[tokio::test]
async fn same_name_rotation_uses_put_and_returns_its_response() {
async fn same_name_rotation_uses_put_and_returns_its_response(
default_settings: KeyManagementSettings,
) {
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.PutSecretValue"))
.and(body_partial_json(
@ -121,7 +175,7 @@ async fn same_name_rotation_uses_put_and_returns_its_response() {
.expect(1)
.mount(&server)
.await;
let response = manager(&server, KeyManagementSettings::default())
let response = manager(&server, default_settings)
.async_rotate_secret("key", "key", &SecretValue::new("replacement"))
.await
.unwrap();
@ -132,8 +186,11 @@ async fn same_name_rotation_uses_put_and_returns_its_response() {
assert_eq!(server.received_requests().await.unwrap().len(), 1);
}
#[rstest]
#[tokio::test]
async fn renamed_rotation_reads_creates_verifies_then_deletes() {
async fn renamed_rotation_reads_creates_verifies_then_deletes(
default_settings: KeyManagementSettings,
) {
let server = MockServer::start().await;
let step = AtomicUsize::new(0);
Mock::given(wiremock::matchers::method("POST"))
@ -176,7 +233,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() {
.mount(&server)
.await;
assert!(matches!(
manager(&server, KeyManagementSettings::default())
manager(&server, default_settings)
.async_rotate_secret("old", "new", &SecretValue::new("replacement"))
.await
.unwrap(),
@ -184,6 +241,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() {
));
}
#[rstest]
#[tokio::test]
async fn creation_passes_tags_and_kms_and_survives_replication_failure() {
let server = MockServer::start().await;
@ -230,6 +288,112 @@ async fn creation_passes_tags_and_kms_and_survives_replication_failure() {
);
}
#[rstest]
#[tokio::test]
async fn trait_write_uses_typed_write_context(default_settings: KeyManagementSettings) {
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.CreateSecret"))
.and(body_partial_json(json!({
"Name": "key",
"SecretString": "value",
"Description": "created by caller",
"Tags": [{"Key": "stage", "Value": "test"}],
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"})))
.expect(1)
.mount(&server)
.await;
let context = SecretWriteContext {
description: Some("created by caller".into()),
tags: std::collections::BTreeMap::from([("stage".into(), "test".into())]),
..Default::default()
};
let response = BaseSecretManager::async_write_secret(
&manager(&server, default_settings),
"key",
&SecretValue::new("value"),
&context,
)
.await
.unwrap();
assert_eq!(response.name(), Some("key"));
}
#[rstest]
#[tokio::test]
async fn trait_delete_accepts_an_unspecified_recovery_window(
default_settings: KeyManagementSettings,
) {
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.DeleteSecret"))
.and(body_partial_json(json!({"SecretId": "key"})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"})))
.expect(1)
.mount(&server)
.await;
let response = BaseSecretManager::async_delete_secret(
&manager(&server, default_settings),
"key",
None,
&SecretOperationContext::default(),
)
.await
.unwrap();
assert_eq!(response.name(), Some("key"));
}
#[rstest]
#[tokio::test]
async fn trait_read_uses_the_aws_region_from_its_operation_context() {
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.respond_with(|request: &wiremock::Request| {
let authorization = request
.headers
.get("authorization")
.unwrap()
.to_str()
.unwrap();
assert!(authorization.contains("/us-west-2/secretsmanager/aws4_request"));
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "value"}))
})
.expect(1)
.mount(&server)
.await;
let context = SecretOperationContext::Aws(AwsOperationContext {
region_name: Some("us-west-2".into()),
..Default::default()
});
let value = BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context)
.await
.unwrap();
assert_eq!(value.unwrap().expose(), "value");
}
#[rstest]
#[tokio::test]
async fn trait_read_applies_the_aws_operation_timeout() {
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.respond_with(
ResponseTemplate::new(200)
.set_delay(Duration::from_secs(1))
.set_body_json(json!({"SecretString": "late"})),
)
.expect(1)
.mount(&server)
.await;
let context = SecretOperationContext::Aws(AwsOperationContext {
timeout: Some(Duration::from_millis(30)),
..Default::default()
});
assert!(matches!(
BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context).await,
Err(Error::Timeout)
));
}
#[rstest]
#[tokio::test]
async fn credential_failures_are_not_swallowed_as_missing_secrets() {
use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
@ -260,9 +424,9 @@ async fn credential_failures_are_not_swallowed_as_missing_secrets() {
assert!(server.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[tokio::test]
async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() {
use std::time::Duration;
let server = MockServer::start().await;
Mock::given(wiremock::matchers::method("POST"))
.respond_with(
@ -291,12 +455,16 @@ async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() {
));
}
#[rstest::rstest]
#[rstest]
#[case::denied(400, "AccessDeniedException")]
#[case::throttled(400, "ThrottlingException")]
#[case::unavailable(503, "ServiceUnavailableException")]
#[tokio::test]
async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str) {
async fn service_failures_remain_errors(
default_settings: KeyManagementSettings,
#[case] status: u16,
#[case] code: &str,
) {
let server = MockServer::start().await;
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
.respond_with(ResponseTemplate::new(status).set_body_json(json!({"__type":code})))
@ -304,7 +472,7 @@ async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str)
.mount(&server)
.await;
assert!(matches!(
manager(&server, KeyManagementSettings::default())
manager(&server, default_settings)
.async_read_secret("key")
.await,
Err(Error::Read(_))

View file

@ -76,10 +76,7 @@ impl AzureKeyVault {
.unwrap_or_default()
}
pub async fn get_secret_from_azure_key_vault(
&self,
name: &str,
) -> Result<Option<Secret>, Error> {
pub async fn get_secret(&self, name: &str) -> Result<Option<Secret>, Error> {
let token = self
.auth
.get_azure_ad_token(&self.inputs, &|key| self.environment.get(key))

View file

@ -2,6 +2,7 @@ use std::sync::Arc;
use litellm_secrets_azure::{AzureKeyVault, Error};
use litellm_secrets_types::{Secret, SecretValue};
use rstest::{fixture, rstest};
use serde::Deserialize;
use wiremock::{
Mock, MockServer, ResponseTemplate,
@ -17,6 +18,7 @@ fn manager(server: &MockServer) -> AzureKeyVault {
.unwrap()
}
#[rstest]
#[tokio::test]
async fn reads_secret_with_bearer_token_and_api_version() {
let server = MockServer::start().await;
@ -32,7 +34,7 @@ async fn reads_secret_with_bearer_token_and_api_version() {
.await;
let secret = manager(&server)
.get_secret_from_azure_key_vault("OPENAI-API-KEY")
.get_secret("OPENAI-API-KEY")
.await
.unwrap()
.unwrap();
@ -40,6 +42,7 @@ async fn reads_secret_with_bearer_token_and_api_version() {
assert_eq!(secret, Secret::String(SecretValue::new("s3cret")));
}
#[rstest]
#[tokio::test]
async fn percent_encodes_secret_name_path_segment() {
let server = MockServer::start().await;
@ -53,7 +56,7 @@ async fn percent_encodes_secret_name_path_segment() {
.await;
let secret = manager(&server)
.get_secret_from_azure_key_vault("name/with spaces")
.get_secret("name/with spaces")
.await
.unwrap()
.unwrap();
@ -61,9 +64,12 @@ async fn percent_encodes_secret_name_path_segment() {
assert_eq!(secret.as_str(), Some("value"));
}
#[rstest::rstest]
#[rstest]
#[case::not_found(404, None)]
#[case::unauthorized(401, Some(401))]
#[case::forbidden(403, Some(403))]
#[case::throttled(429, Some(429))]
#[case::server_error(500, Some(500))]
#[tokio::test]
async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option<u16>) {
let server = MockServer::start().await;
@ -73,9 +79,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option<u
.mount(&server)
.await;
let result = manager(&server)
.get_secret_from_azure_key_vault("NAME")
.await;
let result = manager(&server).get_secret("NAME").await;
match expected_status {
None => assert_eq!(result.unwrap(), None),
@ -83,6 +87,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option<u
}
}
#[rstest]
#[tokio::test]
async fn missing_value_is_an_error() {
let server = MockServer::start().await;
@ -93,41 +98,43 @@ async fn missing_value_is_an_error() {
.await;
assert!(matches!(
manager(&server)
.get_secret_from_azure_key_vault("NAME")
.await,
manager(&server).get_secret("NAME").await,
Err(Error::MissingValue)
));
}
#[test]
fn new_validates_vault_environment() {
assert!(matches!(
AzureKeyVault::new(Arc::new(|_: &str| None)),
Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI"))
));
assert!(matches!(
AzureKeyVault::new(Arc::new(|name: &str| {
(name == "AZURE_KEY_VAULT_URI").then(|| "http://vault.example".to_owned())
})),
Err(Error::VaultUri)
));
assert!(matches!(
AzureKeyVault::new(Arc::new(|name: &str| {
(name == "AZURE_KEY_VAULT_URI").then(|| "vault.example".to_owned())
})),
Err(Error::VaultUri)
));
#[rstest]
#[case::missing(None, true)]
#[case::http(Some("http://vault.example"), false)]
#[case::relative(Some("vault.example"), false)]
#[case::malformed(Some("://"), false)]
fn new_validates_vault_environment(
#[case] uri: Option<&'static str>,
#[case] missing_environment: bool,
) {
let result = AzureKeyVault::new(Arc::new(move |name: &str| {
(name == "AZURE_KEY_VAULT_URI")
.then(|| uri.map(str::to_owned))
.flatten()
}));
if missing_environment {
assert!(matches!(
result,
Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI"))
));
} else {
assert!(matches!(result, Err(Error::VaultUri)));
}
}
#[rstest::rstest]
#[case("https://myvault.vault.azure.net", "https://vault.azure.net/.default")]
#[case(
#[rstest]
#[case::public_cloud("https://myvault.vault.azure.net", "https://vault.azure.net/.default")]
#[case::government_cloud(
"https://v.vault.usgovcloudapi.net/",
"https://vault.usgovcloudapi.net/.default"
)]
#[case("http://localhost:8080", "https://localhost/.default")]
#[test]
#[case::local("http://localhost:8080", "https://localhost/.default")]
fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) {
let manager = AzureKeyVault::with_client(
reqwest::Client::new(),
@ -139,6 +146,7 @@ fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) {
assert_eq!(manager.scope(), expected);
}
#[rstest]
#[tokio::test]
async fn missing_credentials_do_not_request_vault() {
let server = MockServer::start().await;
@ -150,7 +158,7 @@ async fn missing_credentials_do_not_request_vault() {
assert!(
manager_without_credentials(&server)
.get_secret_from_azure_key_vault("NAME")
.get_secret("NAME")
.await
.is_err()
);
@ -192,11 +200,15 @@ struct FixtureExpected {
error: Option<bool>,
}
#[fixture]
fn parity_fixture() -> Fixture {
serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap()
}
#[rstest]
#[tokio::test]
async fn parity_fixture_matches_python_backend_contract() {
let fixture: Fixture =
serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap();
for case in fixture.cases {
async fn parity_fixture_matches_python_backend_contract(parity_fixture: Fixture) {
for case in parity_fixture.cases {
let server = MockServer::start().await;
Mock::given(path(format!("/secrets/{}", case.secret_name)))
.respond_with(
@ -205,9 +217,7 @@ async fn parity_fixture_matches_python_backend_contract() {
.expect(1)
.mount(&server)
.await;
let result = manager(&server)
.get_secret_from_azure_key_vault(&case.secret_name)
.await;
let result = manager(&server).get_secret(&case.secret_name).await;
if case.expected.missing == Some(true) {
assert_eq!(result.unwrap(), None);
} else if case.expected.error == Some(true) {

View file

@ -3,18 +3,16 @@ use std::sync::Arc;
use litellm_core_utils::settings::ProcessEnvironment;
use litellm_secrets_azure::AzureKeyVault;
use litellm_secrets_types::Secret;
use rstest::rstest;
#[rstest]
#[tokio::test]
#[ignore]
async fn reads_a_real_secret() {
let environment = Arc::new(ProcessEnvironment);
let manager = AzureKeyVault::new(environment).unwrap();
let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap();
let secret = manager
.get_secret_from_azure_key_vault(&name)
.await
.unwrap()
.unwrap();
let secret = manager.get_secret(&name).await.unwrap().unwrap();
assert!(matches!(&secret, Secret::String(_)));
let host = std::env::var("AZURE_KEY_VAULT_URI")
.unwrap()

View file

@ -2,7 +2,10 @@ use std::{fs, sync::Arc, time::Duration};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_core_utils::settings::Lookup;
use litellm_secrets_types::{BaseSecretManager, SecretValue, validate_secret_name};
use litellm_secrets_types::{
BaseSecretManager, SecretOperationContext, SecretValue, SecretWriteContext,
validate_secret_name,
};
use moka::future::Cache;
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode};
@ -140,7 +143,7 @@ impl CyberArkSecretManager {
.map_err(|_| Error::Endpoint)
}
async fn authenticate(&self) -> Result<SecretValue, Error> {
async fn authenticate(&self, context: &SecretOperationContext) -> Result<SecretValue, Error> {
if let Some(token) = self.token.get(&()).await {
return Ok(token);
}
@ -155,12 +158,12 @@ impl CyberArkSecretManager {
self.account, self.username
))
.map_err(|_| Error::Endpoint)?;
let response = self
.client
.post(url)
.body(self.api_key.expose().to_owned())
.send()
.await?;
let response = with_timeout(
self.client.post(url).body(self.api_key.expose().to_owned()),
context,
)
.send()
.await?;
if !response.status().is_success() {
return Err(Error::AuthStatus(response.status().as_u16()));
}
@ -169,23 +172,37 @@ impl CyberArkSecretManager {
Ok(token)
}
async fn authorization_header(&self) -> Result<String, Error> {
async fn authorization_header(
&self,
context: &SecretOperationContext,
) -> Result<String, Error> {
Ok(format!(
"Token token=\"{}\"",
self.authenticate().await?.expose()
self.authenticate(context).await?.expose()
))
}
pub async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
self.async_read_secret_with_context(name, &SecretOperationContext::default())
.await
}
pub async fn async_read_secret_with_context(
&self,
name: &str,
context: &SecretOperationContext,
) -> Result<Option<SecretValue>, Error> {
if let Some(value) = self.secrets.get(name).await {
return Ok(Some(value));
}
let response = self
.client
.get(self.secret_url(name)?)
.header("Authorization", self.authorization_header().await?)
.send()
.await?;
let response = with_timeout(
self.client
.get(self.secret_url(name)?)
.header("Authorization", self.authorization_header(context).await?),
context,
)
.send()
.await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
@ -198,20 +215,38 @@ impl CyberArkSecretManager {
}
pub async fn async_write_secret(
&self,
name: &str,
value: &SecretValue,
description: Option<&str>,
) -> Result<(), Error> {
self.async_write_secret_with_context(
name,
value,
description,
&SecretOperationContext::default(),
)
.await
}
pub async fn async_write_secret_with_context(
&self,
name: &str,
value: &SecretValue,
_description: Option<&str>,
context: &SecretOperationContext,
) -> Result<(), Error> {
validate_secret_name(name)?;
self.ensure_variable_exists(name).await;
let response = self
.client
.post(self.secret_url(name)?)
.header("Authorization", self.authorization_header().await?)
.body(value.expose().to_owned())
.send()
.await?;
self.ensure_variable_exists(name, context).await;
let response = with_timeout(
self.client
.post(self.secret_url(name)?)
.header("Authorization", self.authorization_header(context).await?)
.body(value.expose().to_owned()),
context,
)
.send()
.await?;
if !response.status().is_success() {
return Err(Error::Status(response.status().as_u16()));
}
@ -219,7 +254,7 @@ impl CyberArkSecretManager {
Ok(())
}
async fn ensure_variable_exists(&self, name: &str) {
async fn ensure_variable_exists(&self, name: &str, context: &SecretOperationContext) {
let policy_url = self
.endpoint
.join(&format!("policies/{}/policy/root", self.account));
@ -227,7 +262,7 @@ impl CyberArkSecretManager {
tracing::warn!("Could not build CyberArk policy endpoint");
return;
};
let Ok(authorization) = self.authorization_header().await else {
let Ok(authorization) = self.authorization_header(context).await else {
tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
return;
};
@ -235,14 +270,16 @@ impl CyberArkSecretManager {
"- !variable {}\n",
serde_json::to_string(name).expect("serializing a string cannot fail")
);
let response = self
.client
.post(policy_url)
.header("Authorization", authorization)
.header("Content-Type", "application/x-yaml")
.body(body)
.send()
.await;
let response = with_timeout(
self.client
.post(policy_url)
.header("Authorization", authorization)
.header("Content-Type", "application/x-yaml")
.body(body),
context,
)
.send()
.await;
match response {
Ok(response) if response.status().is_success() => {}
Ok(response)
@ -271,7 +308,21 @@ impl CyberArkSecretManager {
pub async fn async_delete_secret(
&self,
name: &str,
_recovery_window_in_days: i64,
recovery_window_in_days: Option<u32>,
) -> Result<DeleteOutcome, Error> {
self.async_delete_secret_with_context(
name,
recovery_window_in_days,
&SecretOperationContext::default(),
)
.await
}
pub async fn async_delete_secret_with_context(
&self,
name: &str,
_recovery_window_in_days: Option<u32>,
_context: &SecretOperationContext,
) -> Result<DeleteOutcome, Error> {
tracing::warn!(
"CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
@ -286,29 +337,50 @@ impl BaseSecretManager for CyberArkSecretManager {
type WriteResponse = ();
type DeleteResponse = DeleteOutcome;
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
self.async_read_secret(name).await
async fn async_read_secret(
&self,
name: &str,
context: &SecretOperationContext,
) -> Result<Option<SecretValue>, Error> {
self.async_read_secret_with_context(name, context).await
}
async fn async_write_secret(
&self,
name: &str,
value: &SecretValue,
description: Option<&str>,
context: &SecretWriteContext,
) -> Result<(), Error> {
self.async_write_secret(name, value, description).await
self.async_write_secret_with_context(
name,
value,
context.description.as_deref(),
&context.operation,
)
.await
}
async fn async_delete_secret(
&self,
name: &str,
recovery_window_in_days: i64,
recovery_window_in_days: Option<u32>,
context: &SecretOperationContext,
) -> Result<DeleteOutcome, Error> {
self.async_delete_secret(name, recovery_window_in_days)
self.async_delete_secret_with_context(name, recovery_window_in_days, context)
.await
}
}
fn with_timeout(
request: reqwest::RequestBuilder,
context: &SecretOperationContext,
) -> reqwest::RequestBuilder {
match context.timeout() {
Some(timeout) => request.timeout(timeout),
None => request,
}
}
fn normalize_endpoint(mut endpoint: reqwest::Url) -> reqwest::Url {
if !endpoint.path().ends_with('/') {
endpoint.set_path(&format!("{}/", endpoint.path()));

View file

@ -2,7 +2,10 @@ use std::{sync::Arc, time::Duration};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_secrets_cyberark::{CyberArkSecretManager, DeleteOutcome, Error};
use litellm_secrets_types::SecretValue;
use litellm_secrets_types::{
BaseSecretManager, CyberarkOperationContext, SecretOperationContext, SecretValue,
};
use rstest::{fixture, rstest};
use serde::Deserialize;
use wiremock::{
Match, Mock, MockServer, Request, ResponseTemplate,
@ -40,7 +43,8 @@ impl Match for RawPath {
}
}
fn fixture() -> ParityFixture {
#[fixture]
fn parity_fixture() -> ParityFixture {
serde_json::from_str(include_str!("fixtures/parity.json")).unwrap()
}
@ -65,6 +69,7 @@ async fn mount_auth(server: &MockServer, expected: u64) {
.await;
}
#[rstest]
#[tokio::test]
async fn successful_reads_cache_auth_secret_and_redact_values() {
let server = MockServer::start().await;
@ -89,6 +94,7 @@ async fn successful_reads_cache_auth_secret_and_redact_values() {
}
}
#[rstest]
#[tokio::test]
async fn concurrent_reads_share_authentication_request() {
let server = MockServer::start().await;
@ -122,7 +128,7 @@ async fn concurrent_reads_share_authentication_request() {
assert_eq!(second.unwrap().unwrap().expose(), "value");
}
#[rstest::rstest]
#[rstest]
#[case::not_found(404)]
#[case::unauthorized(401)]
#[case::forbidden(403)]
@ -162,6 +168,7 @@ async fn failed_reads_are_not_cached(#[case] status: u16) {
}
}
#[rstest]
#[tokio::test]
async fn failed_authentication_is_not_cached_and_does_not_read_secret() {
let server = MockServer::start().await;
@ -199,6 +206,29 @@ async fn failed_authentication_is_not_cached_and_does_not_read_secret() {
);
}
#[rstest]
#[tokio::test]
async fn trait_read_applies_cyberark_operation_timeout_to_authentication() {
let server = MockServer::start().await;
Mock::given(path("/authn/acct/admin/authenticate"))
.respond_with(
ResponseTemplate::new(200)
.set_body_string(TOKEN_JSON)
.set_delay(Duration::from_millis(50)),
)
.mount(&server)
.await;
let manager = manager(&server, Duration::from_secs(60));
let context = SecretOperationContext::Cyberark(CyberarkOperationContext {
timeout: Some(Duration::from_millis(10)),
});
let result = BaseSecretManager::async_read_secret(&manager, "key", &context).await;
assert!(matches!(result, Err(Error::Http(error)) if error.is_timeout()));
}
#[rstest]
#[tokio::test]
async fn expired_tokens_and_secrets_are_fetched_again() {
let server = MockServer::start().await;
@ -215,13 +245,14 @@ async fn expired_tokens_and_secrets_are_fetched_again() {
}
}
#[rstest::rstest]
#[rstest]
#[case::plain("OPENAI_API_KEY")]
#[case::path("team/app/key")]
#[case::punctuation("a b+c.d-e_f~g")]
#[case::quote("needs \"quote\"")]
#[tokio::test]
async fn secret_names_use_python_quote_encoding(
#[values("OPENAI_API_KEY", "team/app/key", "a b+c.d-e_f~g", "needs \"quote\"")] name: &str,
) {
let fixture = fixture();
let secret = fixture
async fn secret_names_use_python_quote_encoding(parity_fixture: ParityFixture, #[case] name: &str) {
let secret = parity_fixture
.secrets
.iter()
.find(|secret| secret.name == name)
@ -244,11 +275,11 @@ async fn secret_names_use_python_quote_encoding(
);
}
#[rstest::rstest]
#[case(201)]
#[case(409)]
#[case(422)]
#[case(500)]
#[rstest]
#[case::created(201)]
#[case::already_exists(409)]
#[case::unprocessable(422)]
#[case::server_error(500)]
#[tokio::test]
async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u16) {
let server = MockServer::start().await;
@ -282,6 +313,7 @@ async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u1
);
}
#[rstest]
#[tokio::test]
async fn failed_value_write_is_not_cached() {
let server = MockServer::start().await;
@ -319,13 +351,17 @@ async fn failed_value_write_is_not_cached() {
);
}
#[rstest]
#[case::parent("../etc")]
#[case::embedded_parent("team/../etc")]
#[case::control("key\n")]
#[tokio::test]
async fn unsafe_names_fail_before_http_calls() {
async fn unsafe_names_fail_before_http_calls(#[case] name: &str) {
let server = MockServer::start().await;
let manager = manager(&server, Duration::from_secs(60));
assert!(matches!(
manager
.async_write_secret("../etc", &SecretValue::new("v"), None)
.async_write_secret(name, &SecretValue::new("v"), None)
.await,
Err(Error::Operation(
litellm_secrets_types::Error::UnsafeSecretName
@ -333,6 +369,7 @@ async fn unsafe_names_fail_before_http_calls() {
));
}
#[rstest]
#[tokio::test]
async fn delete_invalidates_cache_and_reports_not_supported() {
let server = MockServer::start().await;
@ -353,7 +390,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() {
"v"
);
assert_eq!(
manager.async_delete_secret("key", 7).await.unwrap(),
manager.async_delete_secret("key", Some(7)).await.unwrap(),
DeleteOutcome::NotSupported
);
assert_eq!(
@ -367,7 +404,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() {
);
}
#[test]
#[rstest]
fn new_validates_credentials_before_license_and_configuration() {
let empty: Arc<dyn litellm_core_utils::settings::Lookup + Send + Sync> =
Arc::new(|_: &str| None);
@ -413,6 +450,7 @@ fn new_validates_credentials_before_license_and_configuration() {
));
}
#[rstest]
#[tokio::test]
async fn new_reads_environment_defaults_end_to_end() {
let server = MockServer::start().await;
@ -446,7 +484,7 @@ async fn new_reads_environment_defaults_end_to_end() {
);
}
#[test]
#[rstest]
fn new_reports_missing_client_certificate_files() {
assert!(matches!(
CyberArkSecretManager::new(
@ -461,6 +499,7 @@ fn new_reports_missing_client_certificate_files() {
));
}
#[rstest]
#[tokio::test]
async fn trailing_slash_endpoint_preserves_base_path() {
let server = MockServer::start().await;
@ -494,23 +533,25 @@ async fn trailing_slash_endpoint_preserves_base_path() {
);
}
#[test]
fn parity_fixture_matches_authentication_contract() {
let fixture = fixture();
assert_eq!(fixture.endpoint, "http://conjur.test:8080");
assert_eq!(fixture.account, "acct");
assert_eq!(fixture.username, "admin");
assert_eq!(fixture.api_key, "k3y");
assert_eq!(fixture.authenticate_path, "/authn/acct/admin/authenticate");
assert_eq!(fixture.token_json, TOKEN_JSON);
#[rstest]
fn parity_fixture_matches_authentication_contract(parity_fixture: ParityFixture) {
assert_eq!(parity_fixture.endpoint, "http://conjur.test:8080");
assert_eq!(parity_fixture.account, "acct");
assert_eq!(parity_fixture.username, "admin");
assert_eq!(parity_fixture.api_key, "k3y");
assert_eq!(
fixture.authorization_header,
parity_fixture.authenticate_path,
"/authn/acct/admin/authenticate"
);
assert_eq!(parity_fixture.token_json, TOKEN_JSON);
assert_eq!(
parity_fixture.authorization_header,
format!("Token token=\"{}\"", STANDARD.encode(TOKEN_JSON))
);
assert_eq!(fixture.policy_path, "/policies/acct/policy/root");
assert_eq!(fixture.secrets.len(), 4);
assert_eq!(parity_fixture.policy_path, "/policies/acct/policy/root");
assert_eq!(parity_fixture.secrets.len(), 4);
assert_eq!(
fixture.secrets[1].policy_body,
parity_fixture.secrets[1].policy_body,
"- !variable \"team/app/key\"\n"
);
}

View file

@ -1,11 +1,14 @@
use base64::{Engine, engine::general_purpose::STANDARD};
use google_cloud_kms_v1::client::KeyManagementService;
use litellm_secrets_google::GoogleKms;
use litellm_core_utils::settings::Lookup;
use litellm_secrets_google::{Error, GoogleKms, kms::validate_environment};
use rstest::rstest;
use wiremock::{
Mock, MockServer, ResponseTemplate,
matchers::{body_json, path},
};
#[rstest]
#[tokio::test]
async fn google_kms_decrypts_using_the_configured_resource() {
let server = MockServer::start().await;
@ -35,15 +38,49 @@ async fn google_kms_decrypts_using_the_configured_resource() {
);
}
#[rstest]
#[case::unset(None)]
#[case::disabled(Some(false))]
#[tokio::test]
async fn disabled_google_kms_loader_does_not_require_environment_configuration() {
async fn disabled_google_kms_loader_does_not_require_environment_configuration(
#[case] enabled: Option<bool>,
) {
use std::sync::Arc;
for enabled in [None, Some(false)] {
assert!(
litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None))
.await
.unwrap()
.is_none()
);
}
assert!(
litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None))
.await
.unwrap()
.is_none()
);
}
#[rstest]
#[case::credentials_missing(None, None, "GOOGLE_APPLICATION_CREDENTIALS")]
#[case::resource_missing(Some("credentials"), None, "GOOGLE_KMS_RESOURCE_NAME")]
fn enabled_google_kms_requires_all_environment_values(
#[case] credentials: Option<&str>,
#[case] resource: Option<&str>,
#[case] missing: &'static str,
) {
let environment = move |name: &str| match name {
"GOOGLE_APPLICATION_CREDENTIALS" => credentials.map(str::to_owned),
"GOOGLE_KMS_RESOURCE_NAME" => resource.map(str::to_owned),
_ => None,
};
assert!(matches!(
validate_environment(&environment as &dyn Lookup),
Err(Error::MissingEnvironment(name)) if name == missing
));
}
#[rstest]
fn complete_google_kms_environment_is_valid() {
let environment = |name: &str| match name {
"GOOGLE_APPLICATION_CREDENTIALS" => Some("credentials".to_owned()),
"GOOGLE_KMS_RESOURCE_NAME" => Some("resource".to_owned()),
_ => None,
};
assert!(validate_environment(&environment as &dyn Lookup).is_ok());
}

View file

@ -2,6 +2,7 @@ use std::{sync::Arc, time::Duration};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_secrets_google::{Error, GoogleSecretManager};
use rstest::{fixture, rstest};
use wiremock::{
Mock, MockServer, ResponseTemplate,
@ -20,11 +21,17 @@ fn manager(server: &MockServer, always_read: bool, ttl: Duration) -> GoogleSecre
.unwrap()
}
#[rstest::rstest]
#[fixture]
fn default_ttl() -> Duration {
Duration::from_secs(60)
}
#[rstest]
#[case::nonempty("private-value")]
#[case::empty("")]
#[tokio::test]
async fn successful_reads_use_auth_latest_version_and_cache_including_empty_values(
default_ttl: Duration,
#[case] value: &str,
) {
let server = MockServer::start().await;
@ -39,7 +46,7 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu
.expect(1)
.mount(&server)
.await;
let manager = manager(&server, false, Duration::from_secs(60));
let manager = manager(&server, false, default_ttl);
for _ in 0..2 {
assert_eq!(
manager
@ -54,21 +61,44 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu
}
}
#[rstest::rstest]
#[case::not_found(404, serde_json::json!({}))]
#[case::unauthorized(401, serde_json::json!({}))]
#[case::forbidden(403, serde_json::json!({}))]
#[case::throttled(429, serde_json::json!({}))]
#[case::unavailable(503, serde_json::json!({}))]
#[case::missing_payload(200, serde_json::json!({"payload":{}}))]
#[case::invalid_base64(200, serde_json::json!({"payload":{"data":"%%%"}}))]
enum ExpectedReadFailure {
Missing,
Status(u16),
MissingPayload,
Base64,
Utf8,
}
#[rstest]
#[case::not_found(404, serde_json::json!({}), ExpectedReadFailure::Missing)]
#[case::unauthorized(401, serde_json::json!({}), ExpectedReadFailure::Status(401))]
#[case::forbidden(403, serde_json::json!({}), ExpectedReadFailure::Status(403))]
#[case::throttled(429, serde_json::json!({}), ExpectedReadFailure::Status(429))]
#[case::unavailable(503, serde_json::json!({}), ExpectedReadFailure::Status(503))]
#[case::missing_payload(
200,
serde_json::json!({"payload":{}}),
ExpectedReadFailure::MissingPayload
)]
#[case::invalid_base64(
200,
serde_json::json!({"payload":{"data":"%%%"}}),
ExpectedReadFailure::Base64
)]
#[case::invalid_utf8(
200,
serde_json::json!({"payload":{"data":STANDARD.encode([0xff])}}),
ExpectedReadFailure::Utf8
)]
#[tokio::test]
async fn failed_or_missing_reads_are_not_cached(
default_ttl: Duration,
#[case] status: u16,
#[case] body: serde_json::Value,
#[case] expected: ExpectedReadFailure,
) {
let server = MockServer::start().await;
let manager = manager(&server, false, Duration::from_secs(60));
let manager = manager(&server, false, default_ttl);
let failing = Mock::given(path(
"/v1/projects/project/secrets/key/versions/latest:access",
))
@ -77,13 +107,16 @@ async fn failed_or_missing_reads_are_not_cached(
.mount_as_scoped(&server)
.await;
let result = manager.get_secret_from_google_secret_manager("key").await;
match status {
404 => assert_eq!(result.unwrap(), None),
200 => assert!(matches!(
result,
Err(Error::MissingPayload | Error::Base64(_))
)),
status => assert!(matches!(result, Err(Error::Status(actual)) if actual == status)),
match expected {
ExpectedReadFailure::Missing => assert_eq!(result.unwrap(), None),
ExpectedReadFailure::Status(expected) => {
assert!(matches!(result, Err(Error::Status(actual)) if actual == expected));
}
ExpectedReadFailure::MissingPayload => {
assert!(matches!(result, Err(Error::MissingPayload)));
}
ExpectedReadFailure::Base64 => assert!(matches!(result, Err(Error::Base64(_)))),
ExpectedReadFailure::Utf8 => assert!(matches!(result, Err(Error::Utf8))),
}
drop(failing);
Mock::given(path(
@ -109,7 +142,7 @@ async fn failed_or_missing_reads_are_not_cached(
}
}
#[rstest::rstest]
#[rstest]
#[case::always_read(true, Duration::from_secs(60))]
#[case::expired_cache(false, Duration::from_millis(1))]
#[tokio::test]
@ -141,7 +174,7 @@ async fn always_read_and_expired_cache_fetch_again(
}
}
#[test]
#[rstest]
fn google_manager_requires_host_license_and_project_configuration() {
assert!(matches!(
GoogleSecretManager::new(Arc::new(|_: &str| None), false),
@ -155,13 +188,29 @@ fn google_manager_requires_host_license_and_project_configuration() {
));
}
#[rstest::rstest]
#[case("true")]
#[case("null")]
#[case("\"text\"")]
#[case("{\"key\":1}")]
#[rstest]
#[case::provider_specific("GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL")]
#[case::shared("SECRET_MANAGER_REFRESH_INTERVAL")]
fn google_manager_rejects_invalid_refresh_intervals(#[case] variable: &'static str) {
let environment = Arc::new(move |name: &str| match name {
"GOOGLE_SECRET_MANAGER_PROJECT_ID" => Some("project".to_owned()),
name if name == variable => Some("not-a-number".to_owned()),
_ => None,
});
assert!(matches!(
GoogleSecretManager::new(environment, true),
Err(Error::RefreshInterval)
));
}
#[rstest]
#[case::boolean("true")]
#[case::null("null")]
#[case::string("\"text\"")]
#[case::object("{\"key\":1}")]
#[tokio::test]
async fn cache_preserves_raw_values(#[case] raw: &str) {
async fn cache_preserves_raw_values(default_ttl: Duration, #[case] raw: &str) {
let server = MockServer::start().await;
Mock::given(path(
"/v1/projects/project/secrets/key/versions/latest:access",
@ -173,7 +222,7 @@ async fn cache_preserves_raw_values(#[case] raw: &str) {
.expect(1)
.mount(&server)
.await;
let manager = manager(&server, false, Duration::from_secs(60));
let manager = manager(&server, false, default_ttl);
for _ in 0..2 {
assert_eq!(
manager

View file

@ -4,6 +4,8 @@ pub enum Error {
EnterpriseRequired,
#[error("invalid secret name")]
InvalidSecretName(#[from] litellm_secrets_types::Error),
#[error("HashiCorp Vault received an incompatible operation context")]
InvalidOperationContext,
#[error("HashiCorp Vault client failed")]
Client(
#[from]
@ -29,6 +31,8 @@ pub enum Error {
MalformedPayload,
#[error("HashiCorp Vault secret value is not a string")]
NonStringValue,
#[error("HashiCorp Vault operation timed out")]
Timeout,
#[error("invalid HashiCorp Vault refresh interval")]
RefreshInterval,
}

View file

@ -1,13 +1,15 @@
use std::{
collections::HashMap,
fmt,
future::Future,
sync::Arc,
time::{Duration, Instant},
};
use litellm_core_utils::settings::Lookup;
use litellm_secrets_types::{
BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name,
BaseSecretManager, HashicorpOperationContext, SecretOperationContext, SecretValue,
SecretWriteContext, async_rotate_secret, validate_secret_name,
};
use moka::future::Cache;
use rustify::errors::ClientError as RustifyClientError;
@ -31,17 +33,23 @@ struct CachedClient {
expires_at: Option<Instant>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct SecretLocation {
pub namespace: Option<String>,
pub mount: String,
pub path: String,
}
#[derive(Clone, Hash, PartialEq, Eq)]
struct CacheKey {
location: SecretLocation,
data_key: String,
}
#[derive(Clone)]
pub struct HashicorpVault {
config: HashicorpVaultConfig,
cache: Cache<String, SecretValue>,
cache: Cache<CacheKey, SecretValue>,
auth_client: Arc<Mutex<Option<CachedClient>>>,
}
@ -71,7 +79,7 @@ impl HashicorpVault {
if !enterprise_enabled {
return Err(Error::EnterpriseRequired);
}
let cache: Cache<String, SecretValue> = Cache::builder()
let cache: Cache<CacheKey, SecretValue> = Cache::builder()
.max_capacity(CACHE_CAPACITY)
.time_to_live(config.refresh_interval)
.build();
@ -83,9 +91,21 @@ impl HashicorpVault {
}
pub fn secret_location(&self, secret_name: &str) -> Result<SecretLocation, Error> {
self.secret_location_with_context(secret_name, &SecretOperationContext::default())
}
pub fn secret_location_with_context(
&self,
secret_name: &str,
context: &SecretOperationContext,
) -> Result<SecretLocation, Error> {
validate_secret_name(secret_name).map_err(Error::InvalidSecretName)?;
let operation: Option<&HashicorpOperationContext> = hashicorp_context(context)?;
let path: String = [
self.config.path_prefix.clone(),
operation
.and_then(|operation| operation.path_prefix.as_deref())
.and_then(path_component)
.or_else(|| self.config.path_prefix.clone()),
Some(secret_name.to_owned()),
]
.into_iter()
@ -94,7 +114,10 @@ impl HashicorpVault {
.join("/");
Ok(SecretLocation {
namespace: self.config.secret_namespace().map(str::to_owned),
mount: self.config.mount.clone(),
mount: operation
.and_then(|operation| operation.mount.as_deref())
.and_then(path_component)
.unwrap_or_else(|| self.config.mount.clone()),
path,
})
}
@ -104,19 +127,37 @@ impl HashicorpVault {
}
pub async fn async_read_secret(&self, secret_name: &str) -> Result<Option<SecretValue>, Error> {
let location: SecretLocation = self.secret_location(secret_name)?;
let cache_key: String = cache_key(&location);
self.async_read_secret_with_context(secret_name, &SecretOperationContext::default())
.await
}
pub async fn async_read_secret_with_context(
&self,
secret_name: &str,
context: &SecretOperationContext,
) -> Result<Option<SecretValue>, Error> {
let location: SecretLocation = self.secret_location_with_context(secret_name, context)?;
let data_key: String = data_key(context)?;
let cache_key = CacheKey {
location: location.clone(),
data_key: data_key.clone(),
};
if let Some(value) = self.cache.get(&cache_key).await {
return Ok(Some(value));
}
let client: Arc<VaultClient> = self.vault_client().await?;
let data: HashMap<String, Value> =
let data: Option<HashMap<String, Value>> = with_timeout(context, async {
let client: Arc<VaultClient> = self.vault_client().await?;
match kv2::read(client.as_ref(), &location.mount, &location.path).await {
Ok(data) => data,
Err(error) if api_status(&error) == Some(404) => return Ok(None),
Err(error) => return Err(map_api_error(error, ErrorContext::Read)),
};
let Some(value) = data.get("key") else {
Ok(data) => Ok(Some(data)),
Err(error) if api_status(&error) == Some(404) => Ok(None),
Err(error) => Err(map_api_error(error, ErrorContext::Read)),
}
})
.await?;
let Some(data) = data else {
return Ok(None);
};
let Some(value) = data.get(&data_key) else {
return Ok(None);
};
let value: &str = value.as_str().ok_or(Error::NonStringValue)?;
@ -131,11 +172,29 @@ impl HashicorpVault {
value: SecretValue,
description: Option<&str>,
) -> Result<Value, Error> {
let location: SecretLocation = self.secret_location(secret_name)?;
let cache_key: String = cache_key(&location);
let data: HashMap<String, Value> = match description {
self.async_write_secret_with_context(
secret_name,
&value,
&SecretWriteContext {
description: description.map(str::to_owned),
..SecretWriteContext::default()
},
)
.await
}
pub async fn async_write_secret_with_context(
&self,
secret_name: &str,
value: &SecretValue,
context: &SecretWriteContext,
) -> Result<Value, Error> {
let location: SecretLocation =
self.secret_location_with_context(secret_name, &context.operation)?;
let data_key: String = data_key(&context.operation)?;
let data: HashMap<String, Value> = match context.description.as_deref() {
Some(description) => [
("key".to_owned(), Value::String(value.expose().to_owned())),
(data_key, Value::String(value.expose().to_owned())),
(
"description".to_owned(),
Value::String(description.to_owned()),
@ -143,27 +202,41 @@ impl HashicorpVault {
]
.into_iter()
.collect(),
None => [("key".to_owned(), Value::String(value.expose().to_owned()))]
None => [(data_key, Value::String(value.expose().to_owned()))]
.into_iter()
.collect(),
};
let client: Arc<VaultClient> = self.vault_client().await?;
let metadata = kv2::set(client.as_ref(), &location.mount, &location.path, &data)
.await
.map_err(|error| map_api_error(error, ErrorContext::Secret))?;
self.cache.invalidate(&cache_key).await;
let metadata = with_timeout(&context.operation, async {
let client: Arc<VaultClient> = self.vault_client().await?;
kv2::set(client.as_ref(), &location.mount, &location.path, &data)
.await
.map_err(|error| map_api_error(error, ErrorContext::Secret))
})
.await?;
self.cache.invalidate_all();
serde_json::to_value(metadata)
.map_err(|source| Error::Client(ClientError::JsonParseError { source }))
}
pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> {
let location: SecretLocation = self.secret_location(secret_name)?;
let cache_key: String = cache_key(&location);
let client: Arc<VaultClient> = self.vault_client().await?;
kv2::delete_latest(client.as_ref(), &location.mount, &location.path)
self.async_delete_secret_with_context(secret_name, &SecretOperationContext::default())
.await
.map_err(|error| map_api_error(error, ErrorContext::Secret))?;
self.cache.invalidate(&cache_key).await;
}
pub async fn async_delete_secret_with_context(
&self,
secret_name: &str,
context: &SecretOperationContext,
) -> Result<(), Error> {
let location: SecretLocation = self.secret_location_with_context(secret_name, context)?;
with_timeout(context, async {
let client: Arc<VaultClient> = self.vault_client().await?;
kv2::delete_latest(client.as_ref(), &location.mount, &location.path)
.await
.map_err(|error| map_api_error(error, ErrorContext::Secret))
})
.await?;
self.cache.invalidate_all();
Ok(())
}
@ -173,7 +246,23 @@ impl HashicorpVault {
new_name: &str,
value: &SecretValue,
) -> Result<Value, Error> {
async_rotate_secret(self, current_name, new_name, value).await
self.async_rotate_secret_with_context(
current_name,
new_name,
value,
&SecretOperationContext::default(),
)
.await
}
pub async fn async_rotate_secret_with_context(
&self,
current_name: &str,
new_name: &str,
value: &SecretValue,
context: &SecretOperationContext,
) -> Result<Value, Error> {
async_rotate_secret(self, current_name, new_name, value, context).await
}
async fn vault_client(&self) -> Result<Arc<VaultClient>, Error> {
@ -254,25 +343,30 @@ impl BaseSecretManager for HashicorpVault {
type WriteResponse = Value;
type DeleteResponse = ();
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
HashicorpVault::async_read_secret(self, name).await
async fn async_read_secret(
&self,
name: &str,
context: &SecretOperationContext,
) -> Result<Option<SecretValue>, Error> {
HashicorpVault::async_read_secret_with_context(self, name, context).await
}
async fn async_write_secret(
&self,
name: &str,
value: &SecretValue,
description: Option<&str>,
context: &SecretWriteContext,
) -> Result<Value, Error> {
HashicorpVault::async_write_secret(self, name, value.clone(), description).await
HashicorpVault::async_write_secret_with_context(self, name, value, context).await
}
async fn async_delete_secret(
&self,
name: &str,
_recovery_window_in_days: i64,
_recovery_window_in_days: Option<u32>,
context: &SecretOperationContext,
) -> Result<(), Error> {
HashicorpVault::async_delete_secret(self, name).await
HashicorpVault::async_delete_secret_with_context(self, name, context).await
}
}
@ -283,11 +377,42 @@ enum ErrorContext {
Secret,
}
fn cache_key(location: &SecretLocation) -> String {
format!(
"{:?}/{}/{}",
location.namespace, location.mount, location.path
)
fn hashicorp_context(
context: &SecretOperationContext,
) -> Result<Option<&HashicorpOperationContext>, Error> {
match context {
SecretOperationContext::Hashicorp(context) => Ok(Some(context)),
SecretOperationContext::Default => Ok(None),
SecretOperationContext::Aws(_) | SecretOperationContext::Cyberark(_) => {
Err(Error::InvalidOperationContext)
}
}
}
fn path_component(value: &str) -> Option<String> {
let value: &str = value.trim().trim_matches('/');
(!value.is_empty()).then(|| value.to_owned())
}
fn data_key(context: &SecretOperationContext) -> Result<String, Error> {
Ok(hashicorp_context(context)?
.and_then(|context| context.data_key.as_deref())
.map(str::trim)
.filter(|data_key| !data_key.is_empty())
.map(str::to_owned)
.unwrap_or_else(|| "key".to_owned()))
}
async fn with_timeout<T>(
context: &SecretOperationContext,
operation: impl Future<Output = Result<T, Error>>,
) -> Result<T, Error> {
match context.timeout() {
Some(timeout) => tokio::time::timeout(timeout, operation)
.await
.map_err(|_| Error::Timeout)?,
None => operation.await,
}
}
fn identity_for(tls: Option<&TlsCertAuth>) -> Result<Option<Identity>, Error> {

View file

@ -2,7 +2,11 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
use litellm_core_utils::settings::Lookup;
use litellm_secrets_hashicorp::{Error, HashicorpVault, HashicorpVaultConfig};
use litellm_secrets_types::SecretValue;
use litellm_secrets_types::{
AwsOperationContext, BaseSecretManager, CyberarkOperationContext, HashicorpOperationContext,
SecretOperationContext, SecretValue, SecretWriteContext,
};
use rstest::{fixture, rstest};
use serde::Deserialize;
use serde_json::json;
use wiremock::{
@ -69,8 +73,14 @@ fn read_response(data: serde_json::Value) -> serde_json::Value {
})
}
#[fixture]
fn token_values() -> Vec<(&'static str, &'static str)> {
vec![("HCP_VAULT_TOKEN", "token")]
}
#[rstest]
#[tokio::test]
async fn token_reads_use_vault_headers_and_cache_values() {
async fn token_reads_use_vault_headers_and_cache_values(token_values: Vec<(&str, &str)>) {
let server: MockServer = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/v1/secret/data/name"))
@ -81,7 +91,7 @@ async fn token_reads_use_vault_headers_and_cache_values() {
.expect(1)
.mount(&server)
.await;
let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]);
let manager: HashicorpVault = manager(&server, &token_values);
assert_eq!(
manager
@ -109,6 +119,7 @@ async fn token_reads_use_vault_headers_and_cache_values() {
);
}
#[rstest]
#[tokio::test]
async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
let server: MockServer = MockServer::start().await;
@ -138,7 +149,7 @@ async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
assert!(manager.async_read_secret("name").await.unwrap().is_some());
}
#[test]
#[rstest]
fn trailing_address_slashes_are_removed() {
let environment: Arc<dyn Lookup + Send + Sync> = Arc::new(|name: &str| match name {
"HCP_VAULT_ADDR" => Some("http://vault.test:8200///".to_owned()),
@ -159,9 +170,9 @@ fn trailing_address_slashes_are_removed() {
);
}
#[rstest::rstest]
#[case("-1")]
#[case("not-a-number")]
#[rstest]
#[case::negative("-1")]
#[case::not_a_number("not-a-number")]
fn invalid_refresh_intervals_are_rejected(#[case] value: &str) {
let environment: Arc<dyn Lookup + Send + Sync> = Arc::new(move |name: &str| match name {
"HCP_VAULT_REFRESH_INTERVAL" => Some(value.to_owned()),
@ -174,6 +185,7 @@ fn invalid_refresh_intervals_are_rejected(#[case] value: &str) {
));
}
#[rstest]
#[tokio::test]
async fn approle_login_uses_namespace_and_reuses_the_token() {
let server: MockServer = MockServer::start().await;
@ -216,6 +228,7 @@ async fn approle_login_uses_namespace_and_reuses_the_token() {
assert!(manager.async_read_secret("name-2").await.unwrap().is_none());
}
#[rstest]
#[tokio::test]
async fn approle_tokens_expire_after_the_vault_lease() {
let server: MockServer = MockServer::start().await;
@ -246,6 +259,7 @@ async fn approle_tokens_expire_after_the_vault_lease() {
assert!(manager.async_read_secret("second").await.unwrap().is_some());
}
#[rstest]
#[tokio::test]
async fn tls_login_posts_the_role_and_uses_the_client_identity() {
let server: MockServer = MockServer::start().await;
@ -343,21 +357,29 @@ async fn tls_login_posts_the_role_and_uses_the_client_identity() {
assert!(login_bodies.contains(&json!({})));
}
#[rstest::rstest]
#[case::missing(404, json!({"errors": ["missing"]}), 0)]
#[case::malformed(200, json!({"data": "invalid"}), 1)]
#[case::missing_key(200, json!({}), 0)]
#[case::non_string(200, json!({"key": 1}), 2)]
#[derive(Clone, Copy)]
enum ExpectedRead {
Missing,
Malformed,
NonString,
}
#[rstest]
#[case::missing(404, json!({"errors": ["missing"]}), ExpectedRead::Missing)]
#[case::malformed(200, json!({"data": "invalid"}), ExpectedRead::Malformed)]
#[case::missing_key(200, json!({}), ExpectedRead::Missing)]
#[case::non_string(200, json!({"key": 1}), ExpectedRead::NonString)]
#[tokio::test]
async fn read_responses_distinguish_absence_and_malformed_payloads(
token_values: Vec<(&str, &str)>,
#[case] status: u16,
#[case] body: serde_json::Value,
#[case] expected: u8,
#[case] expected: ExpectedRead,
) {
let server: MockServer = MockServer::start().await;
Mock::given(method("GET"))
.respond_with(ResponseTemplate::new(status).set_body_json(
if status == 200 && expected != 1 {
if status == 200 && !matches!(expected, ExpectedRead::Malformed) {
read_response(body)
} else {
body
@ -366,20 +388,19 @@ async fn read_responses_distinguish_absence_and_malformed_payloads(
.expect(1)
.mount(&server)
.await;
let result: Result<Option<SecretValue>, Error> =
manager(&server, &[("HCP_VAULT_TOKEN", "token")])
.async_read_secret("name")
.await;
let result: Result<Option<SecretValue>, Error> = manager(&server, &token_values)
.async_read_secret("name")
.await;
match expected {
0 => assert!(result.unwrap().is_none()),
1 => assert!(matches!(result, Err(Error::MalformedPayload))),
2 => assert!(matches!(result, Err(Error::NonStringValue))),
_ => unreachable!(),
ExpectedRead::Missing => assert!(result.unwrap().is_none()),
ExpectedRead::Malformed => assert!(matches!(result, Err(Error::MalformedPayload))),
ExpectedRead::NonString => assert!(matches!(result, Err(Error::NonStringValue))),
}
}
#[rstest]
#[tokio::test]
async fn write_and_delete_invalidate_the_read_cache() {
async fn write_and_delete_invalidate_the_read_cache(token_values: Vec<(&str, &str)>) {
let server: MockServer = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/v1/secret/data/name"))
@ -418,7 +439,7 @@ async fn write_and_delete_invalidate_the_read_cache() {
.expect(1)
.mount(&server)
.await;
let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]);
let manager: HashicorpVault = manager(&server, &token_values);
assert!(manager.async_read_secret("name").await.unwrap().is_some());
assert!(
@ -431,6 +452,313 @@ async fn write_and_delete_invalidate_the_read_cache() {
manager.async_delete_secret("name").await.unwrap();
}
#[rstest]
#[tokio::test]
async fn base_manager_context_overrides_vault_location_and_data_key(
token_values: Vec<(&str, &str)>,
) {
let server: MockServer = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/v1/alternate/data/managed/name"))
.respond_with(
ResponseTemplate::new(200).set_body_json(read_response(json!({"api_token": "value"}))),
)
.expect(2)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/v1/alternate/data/managed/name"))
.and(body_json(json!({
"data": {"api_token": "updated", "description": "Managed key"}
})))
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
"data": {
"created_time": "",
"deletion_time": "",
"custom_metadata": null,
"destroyed": false,
"version": 2
},
"lease_id": "",
"lease_duration": 0,
"renewable": false,
"request_id": "",
"warnings": null,
"wrap_info": null
})))
.expect(1)
.mount(&server)
.await;
Mock::given(method("DELETE"))
.and(path("/v1/alternate/data/managed/name"))
.respond_with(ResponseTemplate::new(204))
.expect(1)
.mount(&server)
.await;
let manager: HashicorpVault = manager(&server, &token_values);
let operation = SecretOperationContext::Hashicorp(HashicorpOperationContext {
mount: Some(" /alternate/ ".to_owned()),
path_prefix: Some(" /managed/ ".to_owned()),
data_key: Some("api_token".to_owned()),
..HashicorpOperationContext::default()
});
let write_context = SecretWriteContext {
description: Some("Managed key".to_owned()),
operation: operation.clone(),
..SecretWriteContext::default()
};
assert_eq!(
BaseSecretManager::async_read_secret(&manager, "name", &operation)
.await
.unwrap()
.unwrap()
.expose(),
"value"
);
BaseSecretManager::async_write_secret(
&manager,
"name",
&SecretValue::new("updated"),
&write_context,
)
.await
.unwrap();
assert!(
BaseSecretManager::async_read_secret(&manager, "name", &operation)
.await
.unwrap()
.is_some()
);
BaseSecretManager::async_delete_secret(&manager, "name", None, &operation)
.await
.unwrap();
}
#[rstest]
#[tokio::test]
async fn reads_cache_each_data_key_for_the_same_vault_path(token_values: Vec<(&str, &str)>) {
let server: MockServer = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/v1/secret/data/name"))
.respond_with(
ResponseTemplate::new(200).set_body_json(read_response(json!({
"key": "primary",
"alternate": "secondary"
}))),
)
.expect(2)
.mount(&server)
.await;
let manager: HashicorpVault = manager(&server, &token_values);
let alternate = SecretOperationContext::Hashicorp(HashicorpOperationContext {
data_key: Some("alternate".to_owned()),
..HashicorpOperationContext::default()
});
assert_eq!(
manager
.async_read_secret("name")
.await
.unwrap()
.unwrap()
.expose(),
"primary"
);
assert_eq!(
BaseSecretManager::async_read_secret(&manager, "name", &alternate)
.await
.unwrap()
.unwrap()
.expose(),
"secondary"
);
assert_eq!(
manager
.async_read_secret("name")
.await
.unwrap()
.unwrap()
.expose(),
"primary"
);
}
#[rstest]
#[tokio::test]
async fn base_manager_context_timeout_limits_vault_io(token_values: Vec<(&str, &str)>) {
let server: MockServer = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/v1/secret/data/name"))
.respond_with(
ResponseTemplate::new(200)
.set_delay(Duration::from_millis(100))
.set_body_json(read_response(json!({"key": "value"}))),
)
.expect(1)
.mount(&server)
.await;
let manager: HashicorpVault = manager(&server, &token_values);
let context = SecretOperationContext::Hashicorp(HashicorpOperationContext {
timeout: Some(Duration::from_millis(10)),
..HashicorpOperationContext::default()
});
assert!(matches!(
BaseSecretManager::async_read_secret(&manager, "name", &context).await,
Err(Error::Timeout)
));
}
#[rstest]
#[case::aws(SecretOperationContext::Aws(AwsOperationContext::default()))]
#[case::cyberark(SecretOperationContext::Cyberark(CyberarkOperationContext::default()))]
#[tokio::test]
async fn foreign_contexts_cannot_access_vault_secrets(
token_values: Vec<(&str, &str)>,
#[case] context: SecretOperationContext,
#[values(false, true)] cached: bool,
) {
let server = MockServer::start().await;
Mock::given(method("GET"))
.and(path("/v1/secret/data/name"))
.respond_with(
ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
)
.expect(u64::from(cached))
.mount(&server)
.await;
let manager = manager(&server, &token_values);
if cached {
assert!(manager.async_read_secret("name").await.unwrap().is_some());
}
assert!(matches!(
BaseSecretManager::async_read_secret(&manager, "name", &context).await,
Err(Error::InvalidOperationContext)
));
assert!(matches!(
BaseSecretManager::async_write_secret(
&manager,
"name",
&SecretValue::new("replacement"),
&SecretWriteContext {
operation: context.clone(),
..SecretWriteContext::default()
},
)
.await,
Err(Error::InvalidOperationContext)
));
assert!(matches!(
BaseSecretManager::async_delete_secret(&manager, "name", None, &context).await,
Err(Error::InvalidOperationContext)
));
assert!(matches!(
manager
.async_rotate_secret_with_context(
"name",
"new",
&SecretValue::new("replacement"),
&context
)
.await,
Err(Error::InvalidOperationContext)
));
assert_eq!(
server.received_requests().await.unwrap().len(),
usize::from(cached)
);
}
#[rstest]
#[tokio::test]
async fn rotation_applies_timeout_to_each_request(token_values: Vec<(&str, &str)>) {
let server = MockServer::start().await;
let timeout = Duration::from_secs(1);
let delay = timeout / 2;
Mock::given(method("GET"))
.and(path("/v1/alternate/data/managed/current"))
.respond_with(
ResponseTemplate::new(200)
.set_delay(delay)
.set_body_json(read_response(json!({"api_token": "original"}))),
)
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/v1/alternate/data/managed/new"))
.and(body_json(json!({
"data": {"api_token": "replacement", "description": "Rotated from current"}
})))
.respond_with(
ResponseTemplate::new(200)
.set_delay(delay)
.set_body_json(json!({
"data": {
"created_time": "",
"deletion_time": "",
"custom_metadata": null,
"destroyed": false,
"version": 1
},
"lease_id": "",
"lease_duration": 0,
"renewable": false,
"request_id": "",
"warnings": null,
"wrap_info": null
})),
)
.mount(&server)
.await;
Mock::given(method("GET"))
.and(path("/v1/alternate/data/managed/new"))
.respond_with(
ResponseTemplate::new(200)
.set_delay(delay)
.set_body_json(read_response(json!({"api_token": "replacement"}))),
)
.mount(&server)
.await;
Mock::given(method("DELETE"))
.and(path("/v1/alternate/data/managed/current"))
.respond_with(ResponseTemplate::new(204).set_delay(delay))
.mount(&server)
.await;
let manager = manager(&server, &token_values);
let context = SecretOperationContext::Hashicorp(HashicorpOperationContext {
timeout: Some(timeout),
mount: Some("alternate".to_owned()),
path_prefix: Some("managed".to_owned()),
data_key: Some("api_token".to_owned()),
});
manager
.async_rotate_secret_with_context(
"current",
"new",
&SecretValue::new("replacement"),
&context,
)
.await
.unwrap();
let requests = server.received_requests().await.unwrap();
let operations: Vec<_> = requests
.iter()
.map(|request| (request.method.as_str(), request.url.path()))
.collect();
assert_eq!(
operations,
[
("GET", "/v1/alternate/data/managed/current"),
("POST", "/v1/alternate/data/managed/new"),
("GET", "/v1/alternate/data/managed/new"),
("DELETE", "/v1/alternate/data/managed/current"),
]
);
}
#[rstest]
#[tokio::test]
async fn no_auth_and_invalid_names_fail_without_requests() {
let server: MockServer = MockServer::start().await;
@ -447,6 +775,7 @@ async fn no_auth_and_invalid_names_fail_without_requests() {
assert!(server.received_requests().await.unwrap().is_empty());
}
#[rstest]
#[tokio::test]
async fn debug_output_redacts_authentication_values() {
let server: MockServer = MockServer::start().await;
@ -468,14 +797,18 @@ struct ParityCase {
secret_name: String,
}
#[test]
fn configuration_matches_python_parity_fixture() {
let cases: Vec<ParityCase> = serde_json::from_str(include_str!(concat!(
#[fixture]
fn parity_cases() -> Vec<ParityCase> {
serde_json::from_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../tests/test_litellm/secret_managers/hashicorp_vault_parity.json"
)))
.unwrap();
for case in cases {
.unwrap()
}
#[rstest]
fn configuration_matches_python_parity_fixture(parity_cases: Vec<ParityCase>) {
for case in parity_cases {
let values: HashMap<String, String> = case.env.clone();
let environment: Arc<dyn Lookup + Send + Sync> =
Arc::new(move |name: &str| values.get(name).cloned());
@ -521,6 +854,7 @@ fn configuration_matches_python_parity_fixture() {
}
}
#[rstest]
#[tokio::test]
#[ignore]
async fn live_vault_round_trip() {

View file

@ -1,4 +1,4 @@
use crate::{Error, SecretValue};
use crate::{Error, SecretOperationContext, SecretValue, SecretWriteContext};
pub fn validate_secret_name(name: &str) -> Result<(), Error> {
if name.split('/').any(|segment| segment == "..")
@ -20,17 +20,22 @@ pub trait BaseSecretManager {
type WriteResponse;
type DeleteResponse;
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Self::Error>;
async fn async_read_secret(
&self,
name: &str,
context: &SecretOperationContext,
) -> Result<Option<SecretValue>, Self::Error>;
async fn async_write_secret(
&self,
name: &str,
value: &SecretValue,
description: Option<&str>,
context: &SecretWriteContext,
) -> Result<Self::WriteResponse, Self::Error>;
async fn async_delete_secret(
&self,
name: &str,
recovery_window_in_days: i64,
recovery_window_in_days: Option<u32>,
context: &SecretOperationContext,
) -> Result<Self::DeleteResponse, Self::Error>;
}
@ -39,20 +44,31 @@ pub async fn async_rotate_secret<M: BaseSecretManager>(
current_name: &str,
new_name: &str,
value: &SecretValue,
context: &SecretOperationContext,
) -> Result<M::WriteResponse, M::Error> {
if manager.async_read_secret(current_name).await?.is_none() {
if manager
.async_read_secret(current_name, context)
.await?
.is_none()
{
return Err(Error::CurrentSecretMissing.into());
}
let response = manager
.async_write_secret(
new_name,
value,
Some(&format!("Rotated from {current_name}")),
&SecretWriteContext::rotated_from(current_name, context.clone()),
)
.await?;
if manager.async_read_secret(new_name).await?.is_none() {
if manager
.async_read_secret(new_name, context)
.await?
.is_none()
{
return Err(Error::NewSecretMissing.into());
}
manager.async_delete_secret(current_name, 7).await?;
manager
.async_delete_secret(current_name, Some(7), context)
.await?;
Ok(response)
}

View file

@ -0,0 +1,65 @@
use std::{collections::BTreeMap, time::Duration};
use crate::SecretValue;
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub enum SecretOperationContext {
#[default]
Default,
Aws(AwsOperationContext),
Hashicorp(HashicorpOperationContext),
Cyberark(CyberarkOperationContext),
}
impl SecretOperationContext {
pub fn timeout(&self) -> Option<Duration> {
match self {
Self::Default => None,
Self::Aws(context) => context.timeout,
Self::Hashicorp(context) => context.timeout,
Self::Cyberark(context) => context.timeout,
}
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct AwsOperationContext {
pub timeout: Option<Duration>,
pub region_name: Option<String>,
pub role_name: Option<String>,
pub session_name: Option<String>,
pub external_id: Option<SecretValue>,
pub profile_name: Option<String>,
pub web_identity_token: Option<SecretValue>,
pub sts_endpoint: Option<String>,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct HashicorpOperationContext {
pub timeout: Option<Duration>,
pub mount: Option<String>,
pub path_prefix: Option<String>,
pub data_key: Option<String>,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct CyberarkOperationContext {
pub timeout: Option<Duration>,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct SecretWriteContext {
pub description: Option<String>,
pub tags: BTreeMap<String, String>,
pub operation: SecretOperationContext,
}
impl SecretWriteContext {
pub fn rotated_from(current_name: &str, operation: SecretOperationContext) -> Self {
Self {
description: Some(format!("Rotated from {current_name}")),
tags: BTreeMap::new(),
operation,
}
}
}

View file

@ -2,11 +2,16 @@
mod base_secret_manager;
mod config;
mod context;
mod error;
mod value;
pub use base_secret_manager::{BaseSecretManager, async_rotate_secret, validate_secret_name};
pub use config::{AccessMode, KeyManagementSettings, KeyManagementSystem};
pub use context::{
AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext,
SecretOperationContext, SecretWriteContext,
};
pub use error::Error;
pub use litellm_auth_types::SecretValue;
pub use value::Secret;

View file

@ -1,15 +1,20 @@
use litellm_secrets_types::{
AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
};
use rstest::{fixture, rstest};
use serde_json::json;
#[test]
fn config_preserves_defaults_nulls_and_serialized_names() {
let empty: KeyManagementSettings = serde_json::from_value(json!({})).unwrap();
assert_eq!(empty, KeyManagementSettings::default());
assert_eq!(empty.access_mode, AccessMode::ReadOnly);
assert_eq!(empty.store_virtual_keys, Some(false));
assert_eq!(empty.prefix_for_stored_virtual_keys, "litellm/");
#[fixture]
fn default_settings() -> KeyManagementSettings {
serde_json::from_value(json!({})).unwrap()
}
#[rstest]
fn config_preserves_defaults_nulls_and_serialized_names(default_settings: KeyManagementSettings) {
assert_eq!(default_settings, KeyManagementSettings::default());
assert_eq!(default_settings.access_mode, AccessMode::ReadOnly);
assert_eq!(default_settings.store_virtual_keys, Some(false));
assert_eq!(default_settings.prefix_for_stored_virtual_keys, "litellm/");
let configured: KeyManagementSettings = serde_json::from_value(json!({
"hosted_keys": [], "store_virtual_keys": null, "access_mode": "write_only",
"aws_web_identity_token": "private-token", "aws_external_id": "private-id",
@ -29,7 +34,7 @@ fn config_preserves_defaults_nulls_and_serialized_names() {
);
}
#[rstest::rstest]
#[rstest]
#[case::aws_kms("aws_kms", KeyManagementSystem::AwsKms)]
#[case::aws_secret_manager("aws_secret_manager", KeyManagementSystem::AwsSecretManager)]
#[case::google_kms("google_kms", KeyManagementSystem::GoogleKms)]
@ -50,11 +55,32 @@ fn key_management_system_serialization_round_trips(
assert_eq!(serde_json::to_value(system).unwrap(), name);
}
#[test]
fn secret_debug_never_exposes_values() {
assert!(
!format!("{:?}", Secret::String(SecretValue::new("sensitive-value")))
.contains("sensitive-value")
);
assert!(!format!("{:?}", Secret::Bool(true)).contains("true"));
#[rstest]
#[case::read_only(AccessMode::ReadOnly, true)]
#[case::write_only(AccessMode::WriteOnly, false)]
#[case::read_and_write(AccessMode::ReadAndWrite, true)]
fn access_mode_reports_readability(#[case] mode: AccessMode, #[case] expected: bool) {
assert_eq!(mode.readable(), expected);
}
#[rstest]
#[case::string(json!("value"), Secret::String(SecretValue::new("value")))]
#[case::boolean(json!(true), Secret::Bool(true))]
#[case::number(json!(7), Secret::Json(json!(7)))]
#[case::null(json!(null), Secret::Json(json!(null)))]
#[case::array(json!(["value"]), Secret::Json(json!(["value"])))]
#[case::object(json!({"key": "value"}), Secret::Json(json!({"key": "value"})))]
fn secret_conversion_preserves_json_types(
#[case] value: serde_json::Value,
#[case] expected: Secret,
) {
assert_eq!(Secret::from_json(value), expected);
}
#[rstest]
#[case::string(Secret::String(SecretValue::new("sensitive-value")), "sensitive-value")]
#[case::boolean(Secret::Bool(true), "true")]
#[case::json(Secret::Json(json!({"private": "value"})), "private")]
fn secret_debug_never_exposes_values(#[case] secret: Secret, #[case] sensitive: &str) {
assert!(!format!("{secret:?}").contains(sensitive));
}

View file

@ -0,0 +1,110 @@
use std::{collections::BTreeMap, time::Duration};
use litellm_secrets_types::{
AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext,
SecretOperationContext, SecretValue, SecretWriteContext,
};
use rstest::{fixture, rstest};
#[fixture]
fn timeout() -> Duration {
Duration::from_secs(30)
}
#[fixture]
fn aws_context(timeout: Duration) -> SecretOperationContext {
SecretOperationContext::Aws(AwsOperationContext {
timeout: Some(timeout),
region_name: Some("us-west-2".into()),
role_name: Some("role".into()),
external_id: Some(SecretValue::new("external-id")),
web_identity_token: Some(SecretValue::new("web-identity-token")),
..AwsOperationContext::default()
})
}
#[rstest]
fn operation_context_preserves_backend_specific_values_and_redacts_secrets(
aws_context: SecretOperationContext,
timeout: Duration,
) {
assert_eq!(aws_context.timeout(), Some(timeout));
assert_eq!(
aws_context,
SecretOperationContext::Aws(AwsOperationContext {
timeout: Some(timeout),
region_name: Some("us-west-2".into()),
role_name: Some("role".into()),
external_id: Some(SecretValue::new("external-id")),
web_identity_token: Some(SecretValue::new("web-identity-token")),
..AwsOperationContext::default()
})
);
let debug = format!("{aws_context:?}");
assert!(!debug.contains("external-id"));
assert!(!debug.contains("web-identity-token"));
}
#[rstest]
#[case::default(SecretOperationContext::Default, None)]
#[case::aws(
SecretOperationContext::Aws(AwsOperationContext {
timeout: Some(Duration::from_secs(1)),
..AwsOperationContext::default()
}),
Some(Duration::from_secs(1))
)]
#[case::hashicorp(
SecretOperationContext::Hashicorp(HashicorpOperationContext {
timeout: Some(Duration::from_secs(2)),
..HashicorpOperationContext::default()
}),
Some(Duration::from_secs(2))
)]
#[case::cyberark(
SecretOperationContext::Cyberark(CyberarkOperationContext {
timeout: Some(Duration::from_secs(3)),
}),
Some(Duration::from_secs(3))
)]
fn operation_context_reports_each_backend_timeout(
#[case] context: SecretOperationContext,
#[case] expected: Option<Duration>,
) {
assert_eq!(context.timeout(), expected);
}
#[rstest]
fn write_context_keeps_tags_separate_from_the_operation_context() {
let context = SecretWriteContext {
description: Some("Managed virtual key".into()),
tags: BTreeMap::from([("team".into(), "team-id".into())]),
operation: SecretOperationContext::Hashicorp(HashicorpOperationContext {
mount: Some("secret".into()),
path_prefix: Some("teams".into()),
data_key: Some("api_key".into()),
..HashicorpOperationContext::default()
}),
};
assert_eq!(context.tags.get("team"), Some(&"team-id".into()));
assert_eq!(context.operation.timeout(), None);
assert_eq!(
context.operation,
SecretOperationContext::Hashicorp(HashicorpOperationContext {
mount: Some("secret".into()),
path_prefix: Some("teams".into()),
data_key: Some("api_key".into()),
..HashicorpOperationContext::default()
})
);
}
#[rstest]
fn rotation_write_context_preserves_the_operation_context(aws_context: SecretOperationContext) {
let context = SecretWriteContext::rotated_from("current", aws_context.clone());
assert_eq!(context.description.as_deref(), Some("Rotated from current"));
assert!(context.tags.is_empty());
assert_eq!(context.operation, aws_context);
}

View file

@ -1,12 +1,15 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use litellm_secrets_types::{
BaseSecretManager, Error, SecretValue, async_rotate_secret, validate_secret_name,
BaseSecretManager, Error, HashicorpOperationContext, SecretOperationContext, SecretValue,
SecretWriteContext, async_rotate_secret, validate_secret_name,
};
use rstest::{fixture, rstest};
struct Manager {
step: AtomicUsize,
absent_at: Option<usize>,
operation: SecretOperationContext,
}
impl BaseSecretManager for Manager {
@ -14,7 +17,12 @@ impl BaseSecretManager for Manager {
type WriteResponse = &'static str;
type DeleteResponse = ();
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
async fn async_read_secret(
&self,
name: &str,
context: &SecretOperationContext,
) -> Result<Option<SecretValue>, Error> {
assert_eq!(context, &self.operation);
let step = self.step.fetch_add(1, Ordering::SeqCst);
assert_eq!(name, if step == 0 { "old" } else { "new" });
Ok((self.absent_at != Some(step)).then(|| SecretValue::new("value")))
@ -24,35 +32,54 @@ impl BaseSecretManager for Manager {
&self,
name: &str,
value: &SecretValue,
description: Option<&str>,
context: &SecretWriteContext,
) -> Result<Self::WriteResponse, Error> {
assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 1);
assert_eq!(name, "new");
assert_eq!(value.expose(), "replacement");
assert_eq!(description, Some("Rotated from old"));
assert_eq!(context.description.as_deref(), Some("Rotated from old"));
assert!(context.tags.is_empty());
assert_eq!(context.operation, self.operation);
Ok("provider-response")
}
async fn async_delete_secret(
&self,
name: &str,
recovery_window_in_days: i64,
recovery_window_in_days: Option<u32>,
context: &SecretOperationContext,
) -> Result<(), Error> {
assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 3);
assert_eq!(name, "old");
assert_eq!(recovery_window_in_days, 7);
assert_eq!(recovery_window_in_days, Some(7));
assert_eq!(context, &self.operation);
Ok(())
}
}
#[fixture]
fn replacement() -> SecretValue {
SecretValue::new("replacement")
}
#[rstest]
#[case::default(SecretOperationContext::Default)]
#[case::backend_specific(SecretOperationContext::Hashicorp(HashicorpOperationContext {
mount: Some("alternate".to_owned()),
..HashicorpOperationContext::default()
}))]
#[tokio::test]
async fn rotation_verifies_before_deleting_and_returns_provider_response() {
async fn rotation_verifies_before_deleting_and_returns_provider_response(
replacement: SecretValue,
#[case] operation: SecretOperationContext,
) {
let manager = Manager {
step: AtomicUsize::new(0),
absent_at: None,
operation: operation.clone(),
};
assert_eq!(
async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement"))
async_rotate_secret(&manager, "old", "new", &replacement, &operation,)
.await
.unwrap(),
"provider-response"
@ -60,11 +87,12 @@ async fn rotation_verifies_before_deleting_and_returns_provider_response() {
assert_eq!(manager.step.load(Ordering::SeqCst), 4);
}
#[rstest::rstest]
#[rstest]
#[case::current_secret_missing(0, Error::CurrentSecretMissing, 1)]
#[case::new_secret_missing(2, Error::NewSecretMissing, 3)]
#[tokio::test]
async fn missing_old_or_new_value_stops_rotation_before_deletion(
replacement: SecretValue,
#[case] absent_at: usize,
#[case] expected: Error,
#[case] calls: usize,
@ -72,32 +100,57 @@ async fn missing_old_or_new_value_stops_rotation_before_deletion(
let manager = Manager {
step: AtomicUsize::new(0),
absent_at: Some(absent_at),
operation: SecretOperationContext::default(),
};
assert_eq!(
async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement"))
.await
.unwrap_err(),
async_rotate_secret(
&manager,
"old",
"new",
&replacement,
&SecretOperationContext::default(),
)
.await
.unwrap_err(),
expected
);
assert_eq!(manager.step.load(Ordering::SeqCst), calls);
}
#[rstest::rstest]
#[rstest]
#[case::parent("..")]
#[case::parent_prefix("../x")]
#[case::parent_segment("x/../y")]
#[case::parent_suffix("x/..")]
#[case::line_feed("line\n")]
#[case::next_line("\u{85}")]
#[case::line_separator("\u{2028}")]
#[case::paragraph_separator("\u{2029}")]
#[case::parent_prefix("../../../other-app/creds")]
#[case::nested_parent_prefix("litellm/../../secret")]
#[case::parent_segment("foo/../bar")]
#[case::parent_suffix("foo/..")]
#[case::single_parent_prefix("../foo")]
#[case::line_feed("foo\nbar")]
#[case::carriage_return("foo\rbar")]
#[case::tab("foo\tbar")]
#[case::null("foo\0bar")]
#[case::delete("foo\u{7f}bar")]
#[case::next_line("foo\u{85}bar")]
#[case::line_separator("foo\u{2028}bar")]
#[case::paragraph_separator("foo\u{2029}bar")]
fn names_reject_path_traversal_and_control_characters(#[case] name: &str) {
assert_eq!(validate_secret_name(name), Err(Error::UnsafeSecretName));
}
#[rstest::rstest]
#[rstest]
#[case::plain_alias("plain-alias")]
#[case::key_with_digits("my-key-123")]
#[case::service_path("prod/my-service-key")]
#[case::email_path("team/user@example.com")]
#[case::colon("foo: bar")]
#[case::spaced_fragment("foo # bar")]
#[case::query("foo?evil=1")]
#[case::fragment("foo#bar")]
#[case::long_alias(&"a".repeat(500))]
#[case::embedded_double_dot("release-1.0..2")]
#[case::path_separator("folder/key")]
#[case::middle_double_dot("my..key")]
#[case::leading_double_dot("..foo")]
#[case::trailing_double_dot("foo..")]
#[case::version_double_dot("v2.0..1-beta")]
#[case::empty("")]
#[case::three_dots("...")]
fn names_allow_safe_values(#[case] name: &str) {

View file

@ -2,7 +2,10 @@ use std::{future::Future, pin::Pin, sync::Arc};
use litellm_core_utils::settings::Lookup;
use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue};
use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret};
#[cfg(any(feature = "aws", feature = "google"))]
use crate::SecretValue;
pub trait ExternalSecretManager: Send + Sync {
fn system(&self) -> KeyManagementSystem;
@ -17,7 +20,6 @@ pub trait ExternalSecretManager: Send + Sync {
#[derive(Clone)]
pub enum SecretManager {
Local,
External(Arc<dyn ExternalSecretManager>),
#[cfg(feature = "aws")]
AwsKms(crate::aws::AwsKms),
@ -38,7 +40,6 @@ pub enum SecretManager {
impl SecretManager {
pub fn system(&self) -> KeyManagementSystem {
match self {
Self::Local => KeyManagementSystem::Local,
Self::External(manager) => manager.system(),
#[cfg(feature = "aws")]
Self::AwsKms(_) => KeyManagementSystem::AwsKms,
@ -65,10 +66,6 @@ pub async fn get_secret_from_manager(
environment: &(dyn Lookup + Send + Sync),
) -> Result<Option<Secret>, Error> {
match client {
SecretManager::Local => Ok(environment
.get(secret_name)
.map(SecretValue::new)
.map(Secret::String)),
SecretManager::External(manager) => {
manager
.read_secret(secret_name, _settings, environment)
@ -117,10 +114,9 @@ pub async fn get_secret_from_manager(
.map(|value| value.map(Secret::String))
.map_err(Error::from),
#[cfg(feature = "azure")]
SecretManager::AzureKeyVault(client) => client
.get_secret_from_azure_key_vault(secret_name)
.await
.map_err(Error::from),
SecretManager::AzureKeyVault(client) => {
client.get_secret(secret_name).await.map_err(Error::from)
}
#[cfg(feature = "cyberark")]
SecretManager::Cyberark(client) => client
.async_read_secret(secret_name)

View file

@ -1,19 +1,14 @@
use std::sync::Arc;
use litellm_secrets::{
Error, KeyManagementSettings, OidcResolver, Secret, SecretManager, SecretManagerState,
SecretResolver, SecretValue, secret_manager_would_be_consulted,
Error, OidcResolver, Secret, SecretManagerState, SecretResolver, SecretValue,
secret_manager_would_be_consulted,
};
fn resolver(value: Option<&str>, configured: bool) -> SecretResolver {
let state = if configured {
SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default())
} else {
SecretManagerState::default()
};
fn resolver(value: Option<&str>) -> SecretResolver {
let value = value.map(str::to_owned);
SecretResolver::new(
Arc::new(state),
Arc::new(SecretManagerState::default()),
Arc::new(move |_: &str| value.clone()),
OidcResolver::default(),
)
@ -30,9 +25,8 @@ fn resolver(value: Option<&str>, configured: bool) -> SecretResolver {
async fn conversion_is_explicit_and_independent_of_manager_configuration(
#[case] input: &str,
#[case] boolean: Option<bool>,
#[values(false, true)] configured: bool,
) {
let resolver = resolver(Some(input), configured);
let resolver = resolver(Some(input));
assert_eq!(
resolver.get_secret("key", None).await.unwrap(),
Some(Secret::String(SecretValue::new(input)))
@ -62,8 +56,8 @@ async fn conversion_is_explicit_and_independent_of_manager_configuration(
#[rstest::rstest]
#[tokio::test]
async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) {
let missing = resolver(None, configured);
async fn defaults_apply_only_to_absence() {
let missing = resolver(None);
assert_eq!(missing.get_secret("key", None).await.unwrap(), None);
assert_eq!(
missing.get_secret_bool("key", Some(false)).await.unwrap(),
@ -92,7 +86,7 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool)
);
}
assert_eq!(
resolver(Some(""), configured)
resolver(Some(""))
.get_secret_str("key", Some(SecretValue::new("default")))
.await
.unwrap()
@ -103,12 +97,8 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool)
}
#[tokio::test]
async fn prefix_is_removed_once_and_local_manager_is_not_consulted() {
let state = SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default());
assert_eq!(
state.system(),
Some(litellm_secrets::KeyManagementSystem::Local)
);
async fn prefix_is_removed_once_and_resolved_from_environment() {
let state = SecretManagerState::default();
assert!(!secret_manager_would_be_consulted(
&state,
"os.environ/os.environ/KEY"
@ -131,7 +121,7 @@ async fn prefix_is_removed_once_and_local_manager_is_not_consulted() {
#[tokio::test]
async fn resolver_future_can_run_on_a_tokio_worker() {
let resolver = resolver(Some("worker-value"), false);
let resolver = resolver(Some("worker-value"));
let result = tokio::spawn(async move { resolver.get_secret_str("KEY", None).await })
.await
.unwrap()
@ -142,7 +132,9 @@ async fn resolver_future_can_run_on_a_tokio_worker() {
#[cfg(feature = "aws")]
mod aws {
use super::*;
use litellm_secrets::{AccessMode, FailurePolicy, aws::AwsSecretsManagerV2};
use litellm_secrets::{
AccessMode, FailurePolicy, KeyManagementSettings, SecretManager, aws::AwsSecretsManagerV2,
};
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
fn state(server: &MockServer, settings: KeyManagementSettings) -> SecretManagerState {
@ -319,7 +311,9 @@ mod aws {
#[case::failure(503)]
#[tokio::test]
async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) {
use litellm_secrets::{FailurePolicy, google::GoogleSecretManager};
use litellm_secrets::{
FailurePolicy, KeyManagementSettings, SecretManager, google::GoogleSecretManager,
};
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
let server = MockServer::start().await;
Mock::given(method("GET"))

View file

@ -8,6 +8,8 @@ mod scanner;
mod tiktoken;
mod unicode_classes;
use std::sync::Arc;
use byte_level::ByteLevelCounter;
use scanner::{SplitPattern, TiktokenCounter};
@ -15,22 +17,29 @@ pub use error::Error;
enum Encoder {
HuggingFace {
tokenizer: Box<tokenizers::Tokenizer>,
tokenizer: Arc<tokenizers::Tokenizer>,
byte_level: Option<ByteLevelCounter>,
},
Tiktoken(TiktokenCounter),
}
/// A count-only tokenizer. Its model tables are immutable, so one built from an already
/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables.
pub struct FastTokenizer(Encoder);
impl FastTokenizer {
pub fn from_json(json: &str) -> Result<Self, Error> {
let tokenizer = json.parse::<tokenizers::Tokenizer>().map_err(Error::Load)?;
Ok(Self::from_shared(Arc::new(tokenizer)))
}
/// Counts with a Hugging Face model another codec already holds; nothing is re-parsed.
pub fn from_shared(tokenizer: Arc<tokenizers::Tokenizer>) -> Self {
let byte_level = ByteLevelCounter::detect(&tokenizer);
Ok(Self(Encoder::HuggingFace {
tokenizer: Box::new(tokenizer),
Self(Encoder::HuggingFace {
tokenizer,
byte_level,
}))
})
}
pub fn from_cl100k_ranks(ranks: &str) -> Result<Self, Error> {
@ -41,12 +50,36 @@ impl FastTokenizer {
Self::from_ranks(SplitPattern::O200k, ranks)
}
/// `cl100k_base` from ranks another loader already parsed.
pub fn from_cl100k_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::from_pairs(SplitPattern::Cl100k, pairs)
}
/// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks
/// another loader already parsed.
pub fn from_o200k_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::from_pairs(SplitPattern::O200k, pairs)
}
fn from_ranks(split: SplitPattern, ranks: &str) -> Result<Self, Error> {
TiktokenCounter::from_ranks(split, ranks)
.map(Encoder::Tiktoken)
.map(Self)
}
fn from_pairs<'a>(
split: SplitPattern,
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
TiktokenCounter::from_pairs(split, pairs)
.map(Encoder::Tiktoken)
.map(Self)
}
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
match &self.0 {
Encoder::Tiktoken(counter) => Ok(counter.count(text)),

View file

@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter {
impl TiktokenCounter {
pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result<Self, Error> {
Self::new(split, MergeRanks::parse(rank_file)?)
}
pub(super) fn from_pairs<'a>(
split: SplitPattern,
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
) -> Result<Self, Error> {
Self::new(split, MergeRanks::from_pairs(pairs)?)
}
fn new(split: SplitPattern, ranks: MergeRanks) -> Result<Self, Error> {
Ok(Self {
ranks: MergeRanks::parse(rank_file)?,
ranks,
piece_len: split.piece_len(),
unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?,
})

View file

@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
impl MergeRanks {
pub(super) fn parse(text: &str) -> Result<Self, Error> {
let ranks = text
.lines()
.filter(|line| !line.is_empty())
.map(parse_line)
.collect::<Result<FxHashMap<_, _>, _>>()?;
Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line))
}
/// The same table from ranks another loader already parsed.
pub(super) fn from_pairs<'a>(
pairs: impl IntoIterator<Item = (&'a [u8], Rank)>,
) -> Result<Self, Error> {
Self::from_entries(pairs.into_iter().map(|(bytes, rank)| {
if rank == NO_RANK {
return Err(Error::Ranks(format!("rank {rank} is reserved")));
}
Ok((Box::from(bytes), rank))
}))
}
fn from_entries(
entries: impl Iterator<Item = Result<(Box<[u8]>, Rank), Error>>,
) -> Result<Self, Error> {
let ranks = entries.collect::<Result<FxHashMap<_, _>, _>>()?;
if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
}

View file

@ -6,5 +6,6 @@ license.workspace = true
repository.workspace = true
[dependencies]
serde_json.workspace = true
thiserror.workspace = true
tokenizers.workspace = true

View file

@ -6,4 +6,6 @@ pub enum Error {
Load(#[source] tokenizers::Error),
#[error("tokenization failed: {0}")]
Encode(#[source] tokenizers::Error),
#[error("token decoding failed: {0}")]
Decode(#[source] tokenizers::Error),
}

View file

@ -2,22 +2,243 @@
mod error;
pub use error::Error;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
pub use error::Error;
use tokenizers::PostProcessor;
pub use tokenizers::{
AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams,
PaddingStrategy, TruncationDirection, TruncationParams,
};
pub fn encoding_from_json(json: &str) -> Result<Encoding, Error> {
serde_json::from_str(json).map_err(|error| Error::Load(error.into()))
}
pub fn encoding_to_json(encoding: &Encoding) -> Result<String, Error> {
serde_json::to_string(encoding).map_err(|error| Error::Load(error.into()))
}
pub struct HuggingFaceTokenizer {
tokenizer: Arc<tokenizers::Tokenizer>,
special_token_ids: HashSet<u32>,
}
impl HuggingFaceTokenizer {
pub fn from_json(json: &str) -> Result<Self, Error> {
json.parse::<tokenizers::Tokenizer>()
.map(Box::new)
.map(Self)
.map(Self::new)
.map_err(Error::Load)
}
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
let special_token_ids: HashSet<u32> = tokenizer
.get_added_tokens_decoder()
.into_iter()
.filter_map(|(id, token)| token.special.then_some(id))
.collect();
Self {
tokenizer: Arc::new(tokenizer),
special_token_ids,
}
}
/// The parsed model, for a count-only counter to share instead of parsing it again.
pub fn shared(&self) -> Arc<tokenizers::Tokenizer> {
Arc::clone(&self.tokenizer)
}
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
self.0
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.len())
.map_err(Error::Encode)
}
pub fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
self.tokenizer
.encode_fast(text, true)
.map(|encoding| encoding.get_ids().to_vec())
.map_err(Error::Encode)
}
pub fn encode_result<'a>(
&self,
input: EncodeInput<'a>,
add_special_tokens: bool,
fast: bool,
) -> Result<Encoding, Error> {
if fast {
return self
.tokenizer
.encode_fast(input, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_char_offsets(input, add_special_tokens)
.map_err(Error::Encode)
}
pub fn encode_batch_result<'a>(
&self,
inputs: Vec<EncodeInput<'a>>,
add_special_tokens: bool,
fast: bool,
) -> Result<Vec<Encoding>, Error> {
if fast {
return self
.tokenizer
.encode_batch_fast(inputs, add_special_tokens)
.map_err(Error::Encode);
}
self.tokenizer
.encode_batch_char_offsets(inputs, add_special_tokens)
.map_err(Error::Encode)
}
pub fn to_json(&self, pretty: bool) -> Result<String, Error> {
self.tokenizer.to_string(pretty).map_err(Error::Load)
}
pub fn token_to_id(&self, token: &str) -> Option<u32> {
self.tokenizer.token_to_id(token)
}
pub fn id_to_token(&self, id: u32) -> Option<String> {
self.tokenizer.id_to_token(id)
}
pub fn vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
self.tokenizer.get_vocab(with_added_tokens)
}
pub fn vocab_size(&self, with_added_tokens: bool) -> usize {
self.tokenizer.get_vocab_size(with_added_tokens)
}
/// The added tokens by id, in id order.
pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> {
let mut added: Vec<(u32, AddedToken)> = self
.tokenizer
.get_added_tokens_decoder()
.into_iter()
.collect();
added.sort_unstable_by_key(|(id, _)| *id);
added
}
pub fn padding(&self) -> Option<&PaddingParams> {
self.tokenizer.get_padding()
}
pub fn truncation(&self) -> Option<&TruncationParams> {
self.tokenizer.get_truncation()
}
/// How many special tokens the post-processor adds to a single sequence or a pair.
pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize {
self.tokenizer
.get_post_processor()
.map_or(0, |processor| processor.added_tokens(is_pair))
}
pub fn encode_special_tokens(&self) -> bool {
self.tokenizer.get_encode_special_tokens()
}
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
if !skip_special_tokens {
return self.tokenizer.decode(ids, false).map_err(Error::Decode);
}
let filtered_ids: Vec<u32> = ids
.iter()
.copied()
.filter(|id| !self.special_token_ids.contains(id))
.collect();
self.tokenizer
.decode(&filtered_ids, true)
.map_err(Error::Decode)
}
pub fn name(&self) -> &str {
"huggingface"
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn codecs_round_trip_and_skip_special_tokens() {
let json = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
));
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
let ids = tokenizer.encode("<SOS>hello<EOT>").unwrap();
assert!(tokenizer.decode(&ids, false).unwrap().contains("<SOS>"));
assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello");
}
#[test]
fn decode_filters_special_added_tokens() {
let json = r#"{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 1,
"content": "<s>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": null,
"pre_tokenizer": {"type": "Whitespace"},
"post_processor": null,
"decoder": null,
"model": {
"type": "WordLevel",
"vocab": {"<unk>": 0, "<s>": 1, "hello": 2},
"unk_token": "<unk>"
}
}"#;
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("<s>"));
assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("<s>"));
}
#[test]
fn vocabulary_lookups_mirror_the_tokenizers_api() {
let json = include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
));
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
let ids = tokenizer.encode("hello").unwrap();
let token = tokenizer.id_to_token(ids[0]).unwrap();
assert_eq!(tokenizer.token_to_id(&token), Some(ids[0]));
assert_eq!(tokenizer.id_to_token(u32::MAX), None);
assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true));
assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false));
let added = tokenizer.added_tokens_decoder();
assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0));
assert!(added.iter().any(|(_, token)| token.special));
assert!(tokenizer.padding().is_none());
assert!(tokenizer.truncation().is_none());
assert!(!tokenizer.encode_special_tokens());
assert_eq!(
tokenizer.num_special_tokens_to_add(false),
tokenizer.encode("").unwrap().len()
);
}
}

View file

@ -6,5 +6,8 @@ license.workspace = true
repository.workspace = true
[dependencies]
base64.workspace = true
once_cell = "1.21.3"
rustc-hash = "2.1.3"
thiserror.workspace = true
tiktoken-rs.workspace = true

View file

@ -1,27 +1,125 @@
#![forbid(unsafe_code)]
mod error;
mod ranks;
use std::collections::HashSet;
pub use error::UnsupportedTokenizer;
pub use ranks::{LoadError, Vocabulary};
pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE);
pub struct TiktokenTokenizer {
encoder: &'static tiktoken_rs::CoreBPE,
/// Present for encodings built from a rank file; the embedded tiktoken-rs singletons
/// behind [`from_name`](Self::from_name) keep their ranks private.
vocabulary: Option<&'static Vocabulary>,
name: &'static str,
}
impl TiktokenTokenizer {
/// Builds `name` from its packaged rank file (read through `load`), once per process.
/// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does.
pub fn from_cached_ranks(
name: &str,
load: impl FnOnce(&str) -> std::io::Result<String>,
) -> Result<Self, LoadError> {
let (loaded, name) = ranks::load(name, load)?;
Ok(Self {
encoder: &loaded.bpe,
vocabulary: Some(&loaded.vocabulary),
name,
})
}
/// The encodings tiktoken-rs embeds, for hosts without the packaged rank files.
pub fn from_name(name: &str) -> Result<Self, UnsupportedTokenizer> {
let tokenizer = match name {
"cl100k_base" => tiktoken_rs::cl100k_base_singleton(),
"o200k_base" => tiktoken_rs::o200k_base_singleton(),
"o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(),
"p50k_base" => tiktoken_rs::p50k_base_singleton(),
"p50k_edit" => tiktoken_rs::p50k_edit_singleton(),
"r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(),
let (encoder, name) = match name {
"cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"),
"o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"),
"o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"),
"p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"),
"p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"),
"r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"),
"gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"),
_ => return Err(UnsupportedTokenizer(name.to_owned())),
};
Ok(Self(tokenizer))
Ok(Self {
encoder,
vocabulary: None,
name,
})
}
pub fn vocabulary(&self) -> Option<&Vocabulary> {
self.vocabulary
}
pub fn count_tokens(&self, text: &str) -> usize {
self.0.count_ordinary(text)
self.encoder.count_ordinary(text)
}
pub fn encode(&self, text: &str) -> Vec<u32> {
self.encoder.encode_ordinary(text)
}
pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result<Vec<u32>, String> {
let allowed = allowed.iter().map(String::as_str).collect();
self.encoder
.encode(text, &allowed)
.map(|(ids, _)| ids)
.map_err(|error| error.to_string())
}
pub fn special_tokens(&self) -> HashSet<String> {
self.encoder
.special_tokens()
.into_iter()
.map(str::to_owned)
.collect()
}
/// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every
/// token sequence the unstable tail could still become, sorted for a stable order.
pub fn encode_with_unstable(
&self,
text: &str,
allowed: &[String],
) -> (Vec<u32>, Vec<Vec<u32>>) {
let allowed = allowed.iter().map(String::as_str).collect();
let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed);
let mut completions: Vec<Vec<u32>> = completions.into_iter().collect();
completions.sort_unstable();
(stable, completions)
}
pub fn decode_bytes(&self, ids: &[u32]) -> Result<Vec<u8>, String> {
self.encoder
.decode_bytes(ids)
.map_err(|error| error.to_string())
}
pub fn decode(&self, ids: &[u32]) -> Result<String, String> {
self.encoder
.decode_bytes(ids)
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
.map_err(|error| error.to_string())
}
pub fn name(&self) -> &str {
self.name
}
}
pub fn encoding_for_model(model: &str) -> Option<&'static str> {
match tiktoken_rs::tokenizer::get_tokenizer(model)? {
tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"),
tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"),
tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"),
tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"),
tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"),
tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => {
Some("r50k_base")
}
}
}
@ -66,5 +164,75 @@ mod tests {
panic!("unknown encoding must be rejected");
};
assert_eq!(name, "unknown-encoding");
assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2");
}
#[test]
fn codecs_round_trip_named_encodings() {
let encodings = [
"cl100k_base",
"o200k_base",
"o200k_harmony",
"p50k_base",
"p50k_edit",
"r50k_base",
"gpt2",
];
let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"];
for name in encodings {
let tokenizer = TiktokenTokenizer::from_name(name).unwrap();
for text in texts {
assert_eq!(
tokenizer.decode(&tokenizer.encode(text)).unwrap(),
text,
"{name}: {text:?}",
);
}
}
}
#[test]
fn decoding_token_prefixes_replaces_incomplete_utf8() {
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
let reference = tiktoken_rs::cl100k_base_singleton();
let ids = tokenizer.encode("🙂漢字");
for end in 1..=ids.len() {
let bytes = reference.decode_bytes(&ids[..end]).unwrap();
assert_eq!(
tokenizer.decode(&ids[..end]).unwrap(),
String::from_utf8_lossy(&bytes),
);
}
assert!(tokenizer.decode(&[u32::MAX]).is_err());
}
#[test]
fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() {
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
let text = "hello fanta";
let (stable, completions) = tokenizer.encode_with_unstable(text, &[]);
assert!(
text.as_bytes()
.starts_with(&tokenizer.decode_bytes(&stable).unwrap())
);
assert!(!completions.is_empty());
for completion in &completions {
let mut ids = stable.clone();
ids.extend(completion);
assert!(
tokenizer
.decode_bytes(&ids)
.unwrap()
.starts_with(text.as_bytes())
);
}
assert!(completions.windows(2).all(|pair| pair[0] < pair[1]));
}
#[test]
fn encoding_for_model_maps_known_models() {
assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base"));
assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base"));
assert_eq!(encoding_for_model("unknown-model"), None);
}
}

View file

@ -0,0 +1,340 @@
use base64::{Engine, engine::general_purpose::STANDARD};
use once_cell::sync::OnceCell;
use rustc_hash::FxHashMap;
use thiserror::Error;
use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank};
use crate::UnsupportedTokenizer;
const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4";
const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790";
const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15";
const LEGACY_PATTERN: &str =
r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s";
const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s";
static CL100K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static O200K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static HARMONY_ENCODER: OnceCell<Loaded> = OnceCell::new();
static P50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
static EDIT_ENCODER: OnceCell<Loaded> = OnceCell::new();
static R50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built
/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does
/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`).
pub(super) struct Loaded {
pub(super) bpe: CoreBPE,
pub(super) vocabulary: Vocabulary,
}
/// The byte-level vocabulary of a tiktoken encoding.
pub struct Vocabulary {
ranks: FxHashMap<Vec<u8>, Rank>,
special_tokens: FxHashMap<String, Rank>,
max_token_value: Rank,
}
impl Vocabulary {
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
pub fn token_byte_values(&self) -> Vec<Vec<u8>> {
let mut values: Vec<Vec<u8>> = self.ranks.keys().cloned().collect();
values.sort_unstable();
values
}
/// The rank of one whole token: a mergeable piece first, then a special token's text.
pub fn encode_single_token(&self, piece: &[u8]) -> Option<Rank> {
if let Some(rank) = self.ranks.get(piece) {
return Some(*rank);
}
std::str::from_utf8(piece)
.ok()
.and_then(|text| self.special_tokens.get(text).copied())
}
pub fn max_token_value(&self) -> Rank {
self.max_token_value
}
/// Every mergeable token with its rank, for building other tables from one parse.
pub fn ranks(&self) -> impl Iterator<Item = (&[u8], Rank)> + '_ {
self.ranks
.iter()
.map(|(bytes, rank)| (bytes.as_slice(), *rank))
}
/// The special tokens with their ranks, tiktoken's `_special_tokens`.
pub fn special_tokens(&self) -> impl Iterator<Item = (&str, Rank)> + '_ {
self.special_tokens
.iter()
.map(|(token, rank)| (token.as_str(), *rank))
}
pub fn is_special_token(&self, rank: Rank) -> bool {
self.special_tokens.values().any(|special| *special == rank)
}
}
#[derive(Debug, Error)]
pub enum LoadError {
#[error(transparent)]
Unsupported(#[from] UnsupportedTokenizer),
#[error("failed to load tiktoken ranks: {0}")]
Ranks(String),
}
/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays
/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share
/// one cached encoder.
pub(super) fn load(
name: &str,
load_file: impl FnOnce(&str) -> std::io::Result<String>,
) -> Result<(&'static Loaded, &'static str), LoadError> {
let (requested, canonical, file, cache) = match name {
"cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER),
"o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER),
"o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER),
"p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER),
"p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER),
"r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER),
"gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER),
_ => return Err(UnsupportedTokenizer(name.to_owned()).into()),
};
let loaded = cache.get_or_try_init(|| {
let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?;
build(canonical, &ranks)
})?;
Ok((loaded, requested))
}
fn build(name: &str, ranks: &str) -> Result<Loaded, LoadError> {
let parsed = ranks
.lines()
.map(parse_rank)
.collect::<Result<Vec<_>, _>>()?;
let encoder: FxHashMap<_, _> = parsed
.into_iter()
.filter(|(_, rank)| name != "r50k_base" || *rank < 50256)
.collect();
if encoder
.values()
.collect::<std::collections::HashSet<_>>()
.len()
!= encoder.len()
|| (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..]))
{
return Err(LoadError::Ranks("invalid vocabulary ranks".into()));
}
let (pattern, specials): (&str, &[(&str, Rank)]) = match name {
"cl100k_base" => (
CL100K_PATTERN,
&[
("<|endoftext|>", 100257),
("<|fim_prefix|>", 100258),
("<|fim_middle|>", 100259),
("<|fim_suffix|>", 100260),
("<|endofprompt|>", 100276),
],
),
"o200k_base" => (
O200K_BASE_PAT_STR,
&[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)],
),
"o200k_harmony" => (
O200K_BASE_PAT_STR,
&[
("<|startoftext|>", 199998),
("<|endoftext|>", 199999),
("<|reserved_200000|>", 200000),
("<|reserved_200001|>", 200001),
("<|return|>", 200002),
("<|constrain|>", 200003),
("<|reserved_200004|>", 200004),
("<|channel|>", 200005),
("<|start|>", 200006),
("<|end|>", 200007),
("<|message|>", 200008),
("<|reserved_200009|>", 200009),
("<|reserved_200010|>", 200010),
("<|reserved_200011|>", 200011),
("<|call|>", 200012),
],
),
"p50k_edit" => (
LEGACY_PATTERN,
&[
("<|endoftext|>", 50256),
("<|fim_prefix|>", 50281),
("<|fim_middle|>", 50282),
("<|fim_suffix|>", 50283),
],
),
_ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]),
};
let reserved = (200013..=201087)
.filter(|_| name == "o200k_harmony")
.map(|rank| (format!("<|reserved_{rank}|>"), rank));
let special_tokens: FxHashMap<String, Rank> = specials
.iter()
.map(|(token, rank)| ((*token).to_owned(), *rank))
.chain(reserved)
.collect();
let max_token_value = encoder
.values()
.chain(special_tokens.values())
.copied()
.max()
.ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?;
let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern)
.map_err(|error| LoadError::Ranks(error.to_string()))?;
Ok(Loaded {
bpe,
vocabulary: Vocabulary {
ranks: encoder,
special_tokens,
max_token_value,
},
})
}
fn parse_rank(line: &str) -> Result<(Vec<u8>, Rank), LoadError> {
let (token, rank) = line
.split_once(' ')
.ok_or_else(|| LoadError::Ranks("missing rank".into()))?;
let bytes = STANDARD
.decode(token)
.map_err(|error| LoadError::Ranks(error.to_string()))?;
let rank = rank
.parse()
.map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?;
Ok((bytes, rank))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::TiktokenTokenizer;
fn read_packaged_ranks(file: &str) -> std::io::Result<String> {
std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../../litellm/litellm_core_utils/tokenizers")
.join(file),
)
}
#[test]
fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() {
for name in [
"cl100k_base",
"o200k_base",
"o200k_harmony",
"p50k_base",
"p50k_edit",
"r50k_base",
"gpt2",
] {
if name != "gpt2" {
assert!(
TiktokenTokenizer::from_cached_ranks(name, |_| {
Err(std::io::Error::other("unreadable vocabulary"))
})
.is_err()
);
}
let loads = std::sync::atomic::AtomicUsize::new(0);
let barrier = std::sync::Barrier::new(4);
let encoders = std::thread::scope(|scope| {
let tasks: Vec<_> = (0..4)
.map(|_| {
scope.spawn(|| {
barrier.wait();
TiktokenTokenizer::from_cached_ranks(name, |file| {
loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
read_packaged_ranks(file)
})
.unwrap()
})
})
.collect();
tasks
.into_iter()
.map(|task| task.join().unwrap())
.collect::<Vec<_>>()
});
assert_eq!(loads.into_inner(), usize::from(name != "gpt2"));
let actual = &encoders[0];
let expected = TiktokenTokenizer::from_name(name).unwrap();
assert_eq!(actual.special_tokens(), expected.special_tokens());
let specials: Vec<_> = expected.special_tokens().into_iter().collect();
let special_text = specials.join(" ");
assert_eq!(
actual.encode_special(&special_text, &specials).unwrap(),
expected.encode_special(&special_text, &specials).unwrap()
);
for text in [
"",
"café 漢字 ع 🙂",
"a\r\nb\t ",
" hello 123456789",
&special_text,
] {
let ids = expected.encode(text);
assert_eq!(actual.encode(text), ids, "{name}: {text:?}");
assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}");
assert_eq!(
actual.decode_bytes(&ids).unwrap(),
expected.decode_bytes(&ids).unwrap()
);
}
let cached = TiktokenTokenizer::from_cached_ranks(name, |_| {
panic!("reloaded cached vocabulary")
})
.unwrap();
assert_eq!(cached.encode("cached"), expected.encode("cached"));
assert_eq!(cached.name(), name);
assert!(expected.vocabulary().is_none());
assert_vocabulary_lookups(name, actual);
}
}
/// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the
/// encoder itself and against the known vocabulary sizes.
fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) {
let max_token_value = match name {
"cl100k_base" => 100_276,
"o200k_base" => 200_018,
"o200k_harmony" => 201_087,
"p50k_base" => 50_280,
"p50k_edit" => 50_283,
"r50k_base" | "gpt2" => 50_256,
_ => unreachable!("{name}"),
};
let vocabulary = tokenizer.vocabulary().unwrap();
assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}");
let values = vocabulary.token_byte_values();
assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}");
for piece in values.iter().step_by(997) {
let rank = vocabulary.encode_single_token(piece).unwrap();
assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}");
assert!(!vocabulary.is_special_token(rank), "{name}");
}
for (token, rank) in vocabulary.special_tokens() {
assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank));
assert!(vocabulary.is_special_token(rank), "{name}: {token}");
}
assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None);
}
#[test]
fn malformed_ranks_return_errors_instead_of_panicking() {
for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] {
assert!(build("cl100k_base", ranks).is_err());
}
let repeated_rank = (0..=u8::MAX)
.map(|byte| format!("{} 0\n", STANDARD.encode([byte])))
.collect::<String>();
assert!(build("cl100k_base", &repeated_rank).is_err());
}
}

View file

@ -1,6 +1,10 @@
# Token counting
`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface
`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer`
Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing
`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it
The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation
@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t
The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2`
All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend
Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits

View file

@ -32,6 +32,8 @@ pub enum Error {
JsonUtf8(#[source] FromUtf8Error),
#[error("tokenization failed: {0}")]
Encode(#[source] Box<dyn std::error::Error + Send + Sync>),
#[error("token decoding failed: {0}")]
Decode(String),
#[error("token counting task failed: {0}")]
Task(String),
}

View file

@ -1,7 +1,31 @@
use litellm_token_counter_fast::Error as BackendError;
pub use litellm_token_counter_fast::FastTokenizer;
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
pub trait FastCounter: TextCodec {
fn fast_counter(&self) -> Option<FastTokenizer>;
}
#[cfg(feature = "huggingface")]
impl FastCounter for crate::huggingface::HuggingFaceTokenizer {
fn fast_counter(&self) -> Option<FastTokenizer> {
Some(FastTokenizer::from_shared(self.shared()))
}
}
#[cfg(feature = "tiktoken")]
impl FastCounter for crate::tiktoken::TiktokenTokenizer {
fn fast_counter(&self) -> Option<FastTokenizer> {
let vocabulary = self.vocabulary()?;
match self.name() {
"cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()),
"o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()),
_ => return None,
}
.ok()
}
}
impl TokenCounter {
pub fn from_json_fast(tokenizer_json: &str) -> Result<Self, Error> {
@ -39,3 +63,65 @@ impl From<BackendError> for Error {
}
}
}
#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))]
mod tests {
use super::*;
use crate::huggingface::HuggingFaceTokenizer;
use crate::tiktoken::TiktokenTokenizer;
const TEXTS: [&str; 4] = [
"",
"hello world <|endoftext|>",
"café 漢字 ع 🙂 line\r\n indented 123456789",
"<SOS>system<EOT> a\u{301} fi",
];
fn packaged(file: &str) -> String {
std::fs::read_to_string(format!(
"{}/../../../litellm/litellm_core_utils/tokenizers/{file}",
env!("CARGO_MANIFEST_DIR")
))
.unwrap()
}
#[test]
fn fast_counters_derived_from_codecs_count_like_the_codecs() {
let huggingface =
HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap();
let fast = huggingface.fast_counter().unwrap();
for text in TEXTS {
assert_eq!(
fast.count_tokens(text).unwrap(),
Tokenizer::count_tokens(&huggingface, text).unwrap(),
"{text:?}"
);
}
for name in ["cl100k_base", "o200k_base", "o200k_harmony"] {
let tiktoken =
TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap();
let fast = tiktoken.fast_counter().unwrap();
for text in TEXTS {
assert_eq!(
fast.count_tokens(text).unwrap(),
tiktoken.count_tokens(text),
"{name}: {text:?}"
);
}
}
}
#[test]
fn encodings_without_a_fast_scanner_keep_the_codec() {
let tiktoken =
TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap();
assert!(tiktoken.fast_counter().is_none());
assert!(
TiktokenTokenizer::from_name("cl100k_base")
.unwrap()
.fast_counter()
.is_none()
);
}
}

View file

@ -1,7 +1,11 @@
use litellm_token_counter_huggingface::Error as BackendError;
pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
pub use litellm_token_counter_huggingface::{
AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection,
PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json,
encoding_to_json,
};
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
impl TokenCounter {
pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer {
}
}
impl TextCodec for HuggingFaceTokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
HuggingFaceTokenizer::encode(self, text).map_err(Error::from)
}
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from)
}
fn name(&self) -> &str {
HuggingFaceTokenizer::name(self)
}
}
impl From<BackendError> for Error {
fn from(error: BackendError) -> Self {
match error {
BackendError::Load(source) => Self::Load(source),
BackendError::Encode(source) => Self::Encode(source),
BackendError::Decode(source) => Self::Decode(source.to_string()),
}
}
}

View file

@ -20,5 +20,5 @@ pub mod tiktoken;
pub use counter::{InputTokenCount, TokenCounter};
pub use error::Error;
pub use tokenizer::Tokenizer;
pub use tokenizer::{TextCodec, Tokenizer};
pub use types::CountableRequest;

View file

@ -1,7 +1,7 @@
pub use litellm_token_counter_tiktoken::TiktokenTokenizer;
use litellm_token_counter_tiktoken::UnsupportedTokenizer;
use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer};
pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model};
use crate::{Error, TokenCounter, Tokenizer};
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
impl TokenCounter {
pub fn from_tiktoken(encoding: &str) -> Result<Self, Error> {
@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer {
}
}
impl TextCodec for TiktokenTokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
Ok(TiktokenTokenizer::encode(self, text))
}
fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result<String, Error> {
TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string()))
}
fn name(&self) -> &str {
TiktokenTokenizer::name(self)
}
}
impl From<UnsupportedTokenizer> for Error {
fn from(error: UnsupportedTokenizer) -> Self {
Self::UnsupportedTokenizer(error.0)
}
}
impl From<LoadError> for Error {
fn from(error: LoadError) -> Self {
match error {
LoadError::Unsupported(error) => error.into(),
LoadError::Ranks(message) => Self::Ranks(message),
}
}
}

View file

@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync {
fn count_tokens(&self, text: &str) -> Result<usize, Error>;
}
pub trait TextCodec: Tokenizer {
fn encode(&self, text: &str) -> Result<Vec<u32>, Error>;
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error>;
fn name(&self) -> &str;
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -58,7 +58,8 @@ from ._lazy_imports_registry import (
if TYPE_CHECKING:
import httpx
from tiktoken import Encoding
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
def get_litellm_globals() -> dict[str, object]:
@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
# These are special lazy loaders for things that are used internally
# They're separate from the main lazy import system because they have specific use cases
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
_default_encoding: "Encoding | None" = None
def _get_default_encoding() -> "Tokenizer":
from litellm.rust_bridge.tokenizer import get_encoding
def _get_default_encoding() -> "Encoding":
"""
Lazily load and cache the default OpenAI encoding.
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
at `litellm` import time. The encoding is cached after the first import.
This is used internally by utils.py functions that need the encoding but shouldn't
trigger its import during module load.
"""
global _default_encoding
if _default_encoding is None:
from litellm.litellm_core_utils.default_encoding import encoding
_default_encoding = encoding
return _default_encoding
return get_encoding("cl100k_base")
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time

View file

@ -7,6 +7,7 @@
"bash_20250124": null,
"code-execution-2025-08-25": "code-execution-2025-08-25",
"compact-2026-01-12": "compact-2026-01-12",
"compact-2026-09-04": "compact-2026-09-04",
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",

View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
_get_service_tier_cost_key,
calculate_cost_component,
generic_cost_per_token,
get_batch_cost_rates,
get_billable_input_tokens,
get_token_type_cost_breakdown,
parse_prompt_tokens_details,
@ -115,6 +116,7 @@ from litellm.types.utils import (
LlmProviders,
LlmProvidersSet,
ModelInfo,
ModelInfoBase,
PromptTokensDetailsWrapper,
ServiceTier,
StandardBuiltInToolsParams,
@ -322,6 +324,48 @@ class OCRPricing(TypedDict, total=False):
annotation_cost_per_page: ReadOnly[float | None]
_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"})
def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool:
return (
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
or model_info.get("tiered_pricing") is not None
)
def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool:
mode: Final = model_info.get("mode")
return mode is None or mode in _WALL_CLOCK_PRICED_MODES
def _per_second_pricing_cost(
model: str,
custom_llm_provider: str | None,
response_time_ms: float | None,
) -> tuple[float, float] | None:
try:
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
return None
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
return None
input_cost_per_second: Final = model_info.get("input_cost_per_second")
output_cost_per_second: Final = model_info.get("output_cost_per_second")
if input_cost_per_second is None and output_cost_per_second is None:
return None
seconds: Final = (response_time_ms or 0.0) / 1000
verbose_logger.debug(
"For model=%s - input_cost_per_second: %s; output_cost_per_second: %s; response time: %s",
model,
input_cost_per_second,
output_cost_per_second,
response_time_ms,
)
return (input_cost_per_second or 0.0) * seconds, (output_cost_per_second or 0.0) * seconds
def cost_per_token(
model: str = "",
prompt_tokens: int = 0,
@ -448,9 +492,6 @@ def cost_per_token(
if response_cost is not None:
return response_cost[0], response_cost[1]
# given
prompt_tokens_cost_usd_dollar: float = 0
completion_tokens_cost_usd_dollar: float = 0
model_cost_ref: Final = litellm.model_cost
# Only callers that explicitly pass `custom_llm_provider` get the
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
@ -611,6 +652,14 @@ def cost_per_token(
number_of_queries=number_of_queries or 1,
optional_params=(getattr(response, "_hidden_params", None) if response else None),
)
elif (
per_second_cost := _per_second_pricing_cost(
model=model,
custom_llm_provider=custom_llm_provider,
response_time_ms=response_time_ms,
)
) is not None:
return per_second_cost
elif custom_llm_provider == "vertex_ai":
cost_router: Final = google_cost_router(
model=model_without_prefix,
@ -685,12 +734,7 @@ def cost_per_token(
)
else:
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
if (
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
or model_info.get("tiered_pricing") is not None
):
if _has_token_or_tiered_pricing(model_info):
return generic_cost_per_token(
model=model,
usage=usage_block,
@ -698,36 +742,8 @@ def cost_per_token(
service_tier=service_tier,
data_residency=data_residency,
)
input_cost_per_second: Final = model_info.get("input_cost_per_second")
if input_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - input_cost_per_second: %s; response time: %s",
model,
input_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000
output_cost_per_second: Final = model_info.get("output_cost_per_second")
if output_cost_per_second is not None and response_time_ms is not None:
verbose_logger.debug(
"For model=%s - output_cost_per_second: %s; response time: %s",
model,
output_cost_per_second,
response_time_ms,
)
## COST PER SECOND ##
completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000
verbose_logger.debug(
"Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
model,
prompt_tokens_cost_usd_dollar,
completion_tokens_cost_usd_dollar,
)
return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model)
return 0.0, 0.0
def get_replicate_completion_pricing(completion_response: dict, total_time=0.0):
@ -1222,6 +1238,21 @@ def _split_responses_ws_logging_object_by_service_tier(
)
def _response_time_ms_for_cost(
completion_response: object,
litellm_logging_obj: LitellmLoggingObject | None,
total_time: float | None,
) -> float:
stamped: Final = getattr(completion_response, "_response_ms", None)
if isinstance(stamped, (int, float)):
return float(stamped)
if total_time:
return total_time
if litellm_logging_obj is not None:
return litellm_logging_obj.get_response_ms()
return 0.0
def completion_cost(
completion_response: object | None = None,
model: str | None = None,
@ -1443,8 +1474,6 @@ def completion_cost(
prompt_tokens_details = _usage.get("prompt_tokens_details") or {}
cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)
total_time = getattr(completion_response, "_response_ms", 0)
hidden_params = getattr(completion_response, "_hidden_params", None)
if hidden_params is not None:
custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None)
@ -1676,6 +1705,11 @@ def completion_cost(
)
return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
response_time_ms = _response_time_ms_for_cost(
completion_response=completion_response,
litellm_logging_obj=litellm_logging_obj,
total_time=total_time,
)
# Calculate cost based on prompt_tokens, completion_tokens
if (
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
@ -1686,7 +1720,7 @@ def completion_cost(
# see https://replicate.com/pricing
elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost:
# for unmapped replicate model, default to replicate's time tracking logic
return get_replicate_completion_pricing(completion_response, total_time)
return get_replicate_completion_pricing(completion_response, response_time_ms)
if model is None:
raise ValueError(
@ -1718,7 +1752,7 @@ def completion_cost(
prompt_tokens=prompt_tokens or 0,
completion_tokens=completion_tokens or 0,
custom_llm_provider=custom_llm_provider,
response_time_ms=total_time,
response_time_ms=response_time_ms,
region_name=None if explicit_pricing else region_name,
custom_cost_per_second=custom_cost_per_second,
custom_cost_per_token=custom_cost_per_token,
@ -2524,35 +2558,14 @@ def batch_cost_calculator(
if not model_info:
return 0.0, 0.0
input_cost_per_token_batches: Final = model_info.get("input_cost_per_token_batches")
batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider)
input_cost_per_token: Final = model_info.get("input_cost_per_token")
output_cost_per_token_batches: Final = model_info.get("output_cost_per_token_batches")
output_cost_per_token: Final = model_info.get("output_cost_per_token")
total_prompt_cost = 0.0
total_completion_cost = 0.0
if input_cost_per_token_batches is not None:
batch_details: Final = parse_prompt_tokens_details(usage)
audio_tokens, image_tokens, video_tokens = (
batch_details["audio_tokens"],
batch_details["image_tokens"],
batch_details["video_tokens"],
)
modality_rates: Final = (
_batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches),
_batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches),
_batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches),
)
total_prompt_cost = sum(
tokens * rate
for tokens, rate in zip(
(
max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0),
audio_tokens,
image_tokens,
video_tokens,
),
(input_cost_per_token_batches, *modality_rates),
)
if batch_rates.input is not None:
total_prompt_cost = _batch_prompt_cost(
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
)
elif input_cost_per_token:
details: Final = parse_prompt_tokens_details(usage)
@ -2572,8 +2585,8 @@ def batch_cost_calculator(
cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
if output_cost_per_token_batches is not None:
total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
if batch_rates.output is not None:
total_completion_cost = usage.completion_tokens * batch_rates.output
elif output_cost_per_token:
total_completion_cost = (
usage.completion_tokens * (output_cost_per_token) / 2
@ -2587,6 +2600,34 @@ def batch_cost_calculator(
return total_prompt_cost, total_completion_cost
def _batch_prompt_cost(
usage: Usage,
model_info: ModelInfo,
input_rate: float,
cache_read_rate: float | None,
cache_creation_rate: float | None,
) -> float:
details: Final = parse_prompt_tokens_details(usage)
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
audio_tokens, image_tokens, video_tokens = (
details["audio_tokens"],
details["image_tokens"],
details["video_tokens"],
)
text_tokens: Final = max(
(usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0
)
return (
text_tokens * input_rate
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
+ cached_tokens * (cache_read_rate or 0.0)
+ written_tokens * (cache_creation_rate or 0.0)
)
def _attribute_value(obj: object, name: str) -> object:
return getattr(obj, name)

View file

@ -4,13 +4,17 @@ arize AI is OTEL compatible
this file has Arize ai specific helper functions
"""
import math
import os
import random
from collections.abc import Callable, Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_logger
from litellm.integrations.arize import _utils
from litellm.integrations.arize._utils import ArizeOTELAttributes
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.integrations.opentelemetry import _MAX_DYNAMIC_TRACER_PROVIDERS, OpenTelemetry, OpenTelemetryConfig
from litellm.types.integrations.arize import ArizeConfig
from litellm.types.services import ServiceLoggerPayload
from litellm.types.utils import StandardCallbackDynamicParams
@ -26,6 +30,9 @@ else:
Protocol = Any
Span = Any
_SUCCESS_SAMPLING_RATE_VAR: Final = "arize_success_sampling_rate"
_ERROR_SAMPLING_RATE_VAR: Final = "arize_error_sampling_rate"
class ArizeLogger(OpenTelemetry):
"""
@ -36,6 +43,26 @@ class ArizeLogger(OpenTelemetry):
fighting over the global ``opentelemetry.trace`` TracerProvider singleton.
"""
def __init__(
self,
config: OpenTelemetryConfig | None = None,
callback_name: str | None = None,
tracer_provider: object | None = None,
logger_provider: object | None = None,
meter_provider: object | None = None,
max_dynamic_tracer_providers: int = _MAX_DYNAMIC_TRACER_PROVIDERS,
random_draw: Callable[[], float] | None = None,
) -> None:
super().__init__(
config=config,
callback_name=callback_name,
tracer_provider=tracer_provider,
logger_provider=logger_provider,
meter_provider=meter_provider,
max_dynamic_tracer_providers=max_dynamic_tracer_providers,
)
self._random_draw: Final[Callable[[], float]] = random_draw if random_draw is not None else random.random
def _init_tracing(self, tracer_provider):
"""
Override to always create a *private* TracerProvider for Arize.
@ -55,6 +82,72 @@ class ArizeLogger(OpenTelemetry):
self.tracer = provider.get_tracer("litellm")
self.span_kind = SpanKind
def _handle_success(
self,
kwargs: dict[str, object],
response_obj: object,
start_time: datetime,
end_time: datetime,
) -> None:
if not self._should_export(kwargs, _SUCCESS_SAMPLING_RATE_VAR):
return
super()._handle_success(kwargs, response_obj, start_time, end_time)
def _handle_failure(
self,
kwargs: dict[str, object],
response_obj: object,
start_time: datetime,
end_time: datetime,
) -> None:
if not self._should_export(kwargs, _ERROR_SAMPLING_RATE_VAR):
return
super()._handle_failure(kwargs, response_obj, start_time, end_time)
def _sampling_rate_for_request(self, kwargs: Mapping[str, object], var: str) -> float | None:
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
if not isinstance(dynamic_params, Mapping):
return None
value: Final = dynamic_params.get(var)
if value is None or value in ("", "None"):
return None
try:
rate = float(value)
except (TypeError, ValueError):
verbose_logger.warning(
"ArizeLogger: %s value %r is not a number; exporting the request",
var,
value,
)
return None
if not math.isfinite(rate) or not 0.0 <= rate <= 1.0:
verbose_logger.warning(
"ArizeLogger: %s value %r is outside 0.0..1.0; exporting the request",
var,
value,
)
return None
return rate
def _should_export(self, kwargs: dict[str, object], var: str) -> bool:
rate: Final = self._sampling_rate_for_request(kwargs, var)
if rate is None:
return True
otel_internal: Final = self._otel_internal_state(kwargs)
key: Final = f"arize_sampled:{var}"
cached: Final = otel_internal.get(key)
if isinstance(cached, bool):
return cached
sampled: Final = rate > 0.0 and self._random_draw() <= rate
otel_internal[key] = sampled
if not sampled:
verbose_logger.debug(
"ArizeLogger: dropping request, %s rate %r rejected the draw",
var,
rate,
)
return sampled
def _init_otel_logger_on_litellm_proxy(self):
"""
Override: Arize should NOT overwrite the proxy's

View file

@ -1240,6 +1240,25 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# End of Team/Key Based Logging Control Flow
#########################################################
def _otel_internal_state(self, kwargs: dict[str, object]) -> dict[str, object]:
"""Return the request-local ``_otel_internal`` marker dict, creating it if absent."""
litellm_params = kwargs.get("litellm_params")
if not isinstance(litellm_params, dict):
litellm_params = {}
kwargs["litellm_params"] = litellm_params
_metadata = litellm_params.get("metadata")
if not isinstance(_metadata, dict):
_metadata = {}
litellm_params["metadata"] = _metadata
_otel_internal = _metadata.get("_otel_internal")
if not isinstance(_otel_internal, dict):
_otel_internal = {}
_metadata["_otel_internal"] = _otel_internal
return _otel_internal
def _emit_once(self, kwargs: dict, *scope: object) -> bool:
"""Return True the first time this handler is asked to emit a span
for the given (handler, scope) on this kwargs; False on repeats.
@ -1264,20 +1283,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
request-local (kwargs is shared across the sync/async callbacks and
lifecycle hooks for one request).
"""
litellm_params = kwargs.get("litellm_params")
if not isinstance(litellm_params, dict):
litellm_params = {}
kwargs["litellm_params"] = litellm_params
_metadata = litellm_params.get("metadata")
if not isinstance(_metadata, dict):
_metadata = {}
litellm_params["metadata"] = _metadata
_otel_internal = _metadata.get("_otel_internal")
if not isinstance(_otel_internal, dict):
_otel_internal = {}
_metadata["_otel_internal"] = _otel_internal
_otel_internal = self._otel_internal_state(kwargs)
spans_logged = _otel_internal.get("spans_logged")
if not isinstance(spans_logged, dict):

View file

@ -427,7 +427,7 @@ class LLMCallSpanData:
# plain ``.get`` — no repeated ``isinstance`` guards.
raw_response: Final = payload.get("response")
response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {})
choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response)
choices_out: Final = _output_choices(response)
# ``finish_reasons`` is metadata, not content, so derive it from
# ``choices_out`` before gating. The raw message/choice bodies are only
# retained when content capture is enabled (see ``capture_span_content``);
@ -752,20 +752,127 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return (choice,)
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
markdowns: Final = tuple(
text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None
def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
"""The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes."""
return (
_completion_choices(response)
or _responses_choices(response)
or _ocr_choices(response)
or _transcription_choices(response)
or _moderation_choices(response)
or _rerank_choices(response)
or _search_choices(response)
or _image_choices(response)
or _binary_choices(response)
)
if not markdowns:
def _text_choice(content: str, finish_reason: str | None = None) -> _Choice:
message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}
return {"message": message, "finish_reason": finish_reason}
def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]:
return (_text_choice("\n\n".join(parts)),) if parts else ()
def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
return tuple(
_text_choice(text, as_str(choice.get("finish_reason")))
if "message" not in choice and isinstance(text := choice.get("text"), str)
else choice
for choice in _dicts(response.get("choices"))
)
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None)
)
def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
text: Final = response.get("text")
return (_text_choice(text),) if isinstance(text, str) and text else ()
def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(
_moderation_verdict(flagged, result.get("categories"))
for result in _dicts(response.get("results"))
if isinstance(flagged := result.get("flagged"), bool)
)
)
def _moderation_verdict(flagged: bool, categories: object) -> str:
if not flagged:
return "not flagged"
hits: Final = (
tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True)
if isinstance(categories, dict)
else ()
)
return f"flagged: {', '.join(hits)}" if hits else "flagged"
def _rerank_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(
_rerank_line(index, score, result.get("document"))
for result in _dicts(response.get("results"))
if (index := as_int(result.get("index"))) is not None
if (score := as_float(result.get("relevance_score"))) is not None
)
)
def _rerank_line(index: int, score: float, document: object) -> str:
text: Final = as_str((as_str_mapping(document) or {}).get("text"))
return f"[{index}] {score}\n{text}" if text else f"[{index}] {score}"
def _search_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(
line
for result in _dicts(response.get("results"))
if (line := "\n".join(part for key in ("title", "url", "snippet") if (part := as_str(result.get(key)))))
)
)
def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
return _joined_choice(
tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None)
)
def _image_summary(item: Mapping[str, object]) -> str | None:
location: Final = _image_location(item)
if location is None:
return None
revised: Final = as_str(item.get("revised_prompt"))
return f"{revised}\n{location}" if revised else location
def _image_location(item: Mapping[str, object]) -> str | None:
url: Final = as_str(item.get("url"))
if url is not None:
return url
encoded: Final = item.get("b64_json")
if not isinstance(encoded, str):
return None
return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)"
def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
size: Final = as_int(response.get("num_bytes"))
if size is None:
return ()
message: Final[_AssistantMessage] = {
"role": "assistant",
"content": "\n\n".join(markdowns),
"refusal": None,
"tool_calls": None,
}
choice: Final[_Choice] = {"message": message, "finish_reason": None}
return (choice,)
content_type: Final = as_str(response.get("content_type"))
return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),)
def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None:

View file

@ -6,8 +6,9 @@ Core files:
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
- `default_encoding.py`: code for loading the default encoding (tiktoken)
- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.

View file

@ -0,0 +1,180 @@
from __future__ import annotations
import os
import platform
import traceback
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal
from urllib.parse import urlencode
import litellm
from litellm._version import version as litellm_version
from litellm.types.utils import LlmProviders
ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new"
MAX_URL_LENGTH: Final = 6000
MAX_FRAMES: Final = 12
DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK"
NOTICE_PREFIX: Final = "This looks like a bug in LiteLLM rather than in your request."
KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders)
Surface = Literal["sdk", "proxy"]
@dataclass(frozen=True, slots=True)
class BugReport:
surface: Surface
exception_type: str
litellm_frames: tuple[str, ...]
litellm_version: str
python_version: str
call_type: str | None
custom_llm_provider: str | None
stream: bool | None
config_lines: tuple[str, ...]
def bug_report_enabled() -> bool:
return os.getenv(DISABLE_ENV_VAR, "").lower() != "true"
def should_report_bug(exc: object) -> bool:
return bug_report_enabled() and isinstance(exc, BaseException) and getattr(exc, "status_code", None) is None
def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None:
frame_path: Final = Path(frame.filename).resolve()
try:
frame_path.relative_to(package_dir)
relative_path: Final = frame_path.relative_to(package_parent)
except ValueError:
return None
return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}"
def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]:
if exc.__traceback__ is None:
return ()
package_dir: Final = Path(litellm.__file__).resolve().parent
package_parent: Final = package_dir.parent
return tuple(
frame_text
for frame in traceback.extract_tb(exc.__traceback__)
if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None
)[-MAX_FRAMES:]
def allowlisted(value: object, allowed: frozenset[str]) -> str | None:
return value if isinstance(value, str) and value in allowed else None
def build_bug_report(
exc: BaseException,
*,
surface: Surface,
call_type: str | None = None,
custom_llm_provider: object = None,
stream: object = None,
config_lines: tuple[str, ...] = (),
) -> BugReport:
return BugReport(
surface=surface,
exception_type=type(exc).__name__,
litellm_frames=_get_litellm_frames(exc),
litellm_version=litellm_version,
python_version=platform.python_version(),
call_type=call_type,
custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS),
stream=stream if isinstance(stream, bool) else None,
config_lines=config_lines,
)
def _domain(report: BugReport) -> str:
if report.surface == "sdk":
return "Python SDK: the litellm package itself"
if report.custom_llm_provider is not None:
return "LLM translation: a specific provider's request or response"
return "Proxy core: startup, config, health checks, endpoints"
def _title(report: BugReport, frames: tuple[str, ...]) -> str:
location: Final = frames[-1].split(":", 1)[0] if frames else "litellm"
return f"[Bug]: {report.exception_type} in {location}"
def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else ""
stream_line: Final = "" if report.stream is None else f"Stream: {str(report.stream).lower()}\n"
config_block: Final = (
"\nConfig (true/false flags and LiteLLM-defined values only):\n```\n" + "\n".join(config_lines) + "\n```\n"
if config_lines
else ""
)
return (
"Auto-generated by LiteLLM's bug report link. It carries no request data or error text. "
"Please describe what you were doing, and paste the error message from your log below "
"if it contains nothing sensitive.\n\n"
"```\n\n```\n\n"
f"Exception: `{report.exception_type}`\n\n"
f"{frame_block}"
f"Surface: {report.surface}\n"
f"Endpoint / call: {report.call_type or 'unknown'}\n"
f"Provider: {report.custom_llm_provider or 'unknown'}\n"
f"LiteLLM: {report.litellm_version}\n"
f"Python: {report.python_version}\n"
f"{stream_line}"
f"{config_block}"
)
def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
deployment: Final[tuple[tuple[str, str], ...]] = (
(("deployment", "pip / Python SDK"),)
if report.surface == "sdk"
else (("deployment", "Docker"),)
if os.path.exists("/.dockerenv")
else ()
)
fields: Final = (
("template", "bug_report.yml"),
("labels", "bug"),
("title", _title(report, frames)),
("version", report.litellm_version),
("domain", _domain(report)),
("description", _description(report, frames, config_lines)),
) + deployment
return f"{ISSUE_URL_BASE}?{urlencode(fields)}"
def bug_report_issue_url(report: BugReport) -> str:
frames: Final = report.litellm_frames
config_lines: Final = report.config_lines
candidates: Final = (
*((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)),
*((frames[index:], ()) for index in range(1, len(frames) + 1)),
)
return next(
(
url
for candidate_frames, candidate_config in candidates
if len(url := _issue_url(report, candidate_frames, candidate_config)) <= MAX_URL_LENGTH
),
_issue_url(report, (), ()),
)
def strip_bug_report_notice(message: str) -> str:
index: Final = message.find(NOTICE_PREFIX)
if index == -1:
return message
head: Final = message[:index]
return head.removesuffix("\n")
def bug_report_notice(report: BugReport) -> str:
return (
f"{NOTICE_PREFIX} File it with one click "
f"(prefilled, no request data or error text, review before submitting): {bug_report_issue_url(report)}"
)

View file

@ -1,5 +1,4 @@
import os
from pathlib import Path
from typing import Final
import litellm
@ -15,20 +14,6 @@ except (ImportError, AttributeError):
filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers")
CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790"
def cl100k_base_rank_file() -> str:
"""The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines)."""
return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii")
def o200k_base_rank_file() -> str:
"""The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines)."""
return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii")
# Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
# unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
# This keeps tiktoken fully offline-capable by default (see #1071).

View file

@ -10,6 +10,11 @@ import httpx
import litellm
from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
from litellm.litellm_core_utils.bug_report import (
bug_report_notice,
build_bug_report,
should_report_bug,
)
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.types.utils import LlmProviders
@ -2673,7 +2678,21 @@ def exception_type(
)
else:
raise APIConnectionError(
message=f"{original_exception}\n{_redact_string(traceback.format_exc())}",
message=(
f"{original_exception}\n{_redact_string(traceback.format_exc())}"
+ (
"\n"
+ bug_report_notice(
build_bug_report(
original_exception,
surface="sdk",
custom_llm_provider=custom_llm_provider,
)
)
if should_report_bug(original_exception)
else ""
)
),
llm_provider=custom_llm_provider,
model=model,
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request

View file

@ -1,10 +1,24 @@
import re
from collections.abc import Iterator, Mapping
from collections.abc import Generator, Iterator, Mapping
from contextlib import contextmanager
from contextvars import ContextVar
from typing import Any, Final
from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
_CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
"inherited_message_logging_disabled", default=False
)
@contextmanager
def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled)
try:
yield
finally:
_inherited_message_logging_disabled.reset(token)
def iter_client_callback_metadata_dicts(
@ -84,6 +98,8 @@ _supported_callback_params: Final[tuple[str, ...]] = (
"arize_api_key",
"arize_space_key",
"arize_space_id",
"arize_success_sampling_rate",
"arize_error_sampling_rate",
"posthog_api_key",
"posthog_host",
"braintrust_api_key",
@ -143,7 +159,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple
def initialize_standard_callback_dynamic_params(
kwargs: dict | None = None,
kwargs: dict[str, object] | None = None,
) -> StandardCallbackDynamicParams:
"""
Initialize the standard callback dynamic params from the kwargs
@ -179,4 +195,10 @@ def initialize_standard_callback_dynamic_params(
if param in _trusted_overlay_callback_params:
standard_callback_dynamic_params[param] = trusted_value
if _inherited_message_logging_disabled.get():
private_params: Final[StandardCallbackDynamicParams] = {
**standard_callback_dynamic_params,
"turn_off_message_logging": True,
}
return private_params
return standard_callback_dynamic_params

View file

@ -107,6 +107,10 @@ from litellm.litellm_core_utils.redact_messages import (
redact_streaming_responses_for_custom_logger,
should_redact_message_logging,
)
from litellm.litellm_core_utils.served_output_texts import (
SERVED_OUTPUT_TEXTS_KEY,
overlay_served_output_texts,
)
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
@ -383,11 +387,40 @@ _DEPLOYMENT_PRICING_KEYS: Final = (
"output_cost_per_token",
"input_cost_per_token_batches",
"output_cost_per_token_batches",
"input_cost_per_token_above_272k_tokens_batches",
"output_cost_per_token_above_272k_tokens_batches",
"cache_read_input_token_cost_batches",
"cache_read_input_token_cost_above_272k_tokens_batches",
"cache_creation_input_token_cost_batches",
"cache_creation_input_token_cost_above_272k_tokens_batches",
"ocr_cost_per_page",
"ocr_cost_per_page_batches",
"annotation_cost_per_page",
"annotation_cost_per_page_batches",
)
_INPUT_PRICING_KEY_PREFIXES: Final = (
"input_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
)
_OUTPUT_PRICING_KEY_PREFIXES: Final = ("output_cost_per_token",)
_BATCH_PRICING_KEY_SUFFIX: Final = "_batches"
_NO_CARRIED_RATES: Final[Mapping[str, object]] = MappingProxyType({})
def _published_direction(
published: ModelInfo, registered: Mapping[str, object], flat_key: str, prefixes: tuple[str, ...]
) -> Mapping[str, object]:
return MappingProxyType(
{
key: value
for key, value in published.items()
if registered.get(key) is None
and (key == flat_key or (key.startswith(prefixes) and key.endswith(_BATCH_PRICING_KEY_SUFFIX)))
}
)
def deployment_pricing_model_info(model_id: str | None, deployment_model: str | None) -> ModelInfo | None:
@ -399,12 +432,15 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str |
get_model_info fills absent costs with 0, so asking it directly cannot
tell "configured as free" apart from "no pricing configured". A deployment
may declare only one side of its pricing, so the side it leaves out keeps
the model's published rates instead of billing as zero. Ownership is per
token direction: declaring either rate for a direction takes that whole
direction, so a published batch rate can never displace a standard rate
the deployment configured itself. OCR per-page rates count as declared
pricing too; they pass through as registered and ``ocr_batch_cost`` layers
the published rate under each per-page family the deployment leaves out.
the model's published standard rate and every published batch rate for
that direction (flat, long-context tier, cached, cache write) instead of
billing as zero. Ownership is per token direction: declaring the flat
standard or flat batch rate for a direction takes that whole direction, so
a published batch rate can never displace a standard rate the deployment
configured itself. A tier-only override keeps every published rate it left
out. OCR per-page rates count as declared pricing too; they pass through as
registered and ``ocr_batch_cost`` layers the published rate under each
per-page family the deployment leaves out.
"""
if model_id is None:
return None
@ -425,13 +461,22 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str |
registered.get("output_cost_per_token") is not None
or registered.get("output_cost_per_token_batches") is not None
)
if not declares_input:
merged["input_cost_per_token"] = published.get("input_cost_per_token")
merged["input_cost_per_token_batches"] = published.get("input_cost_per_token_batches")
if not declares_output:
merged["output_cost_per_token"] = published.get("output_cost_per_token")
merged["output_cost_per_token_batches"] = published.get("output_cost_per_token_batches")
return merged
carried_input: Final = (
_NO_CARRIED_RATES
if declares_input
else _published_direction(published, registered, "input_cost_per_token", _INPUT_PRICING_KEY_PREFIXES)
)
carried_output: Final = (
_NO_CARRIED_RATES
if declares_output
else _published_direction(published, registered, "output_cost_per_token", _OUTPUT_PRICING_KEY_PREFIXES)
)
priced: Final[ModelInfo] = { # pyright: ignore[reportAssignmentType] # carried keys are ModelInfo rates
**merged,
**carried_input,
**carried_output,
}
return priced
def _published_pricing(deployment_model: str | None) -> ModelInfo | None:
@ -496,6 +541,14 @@ def mask_api_base_credentials(api_base: str) -> str:
return api_base[:key_end] + "*" * 5 + api_base[-4:]
def _timestamp_seconds(moment: object) -> float | None:
if isinstance(moment, datetime.datetime):
return moment.timestamp()
if isinstance(moment, (int, float)):
return float(moment)
return None
class Logging(LiteLLMLoggingBaseClass):
global \
supabaseClient, \
@ -1634,10 +1687,12 @@ class Logging(LiteLLMLoggingBaseClass):
return response.mcp_tool_call_response
def get_response_ms(self) -> float:
return (
self.model_call_details.get("end_time", datetime.datetime.now())
- self.model_call_details.get("start_time", datetime.datetime.now())
).total_seconds() * 1000
now: Final = datetime.datetime.now()
start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now))
end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now))
if start_seconds is None or end_seconds is None:
return 0.0
return (end_seconds - start_seconds) * 1000
def set_cost_breakdown(
self,
@ -5833,15 +5888,12 @@ class StandardLoggingPayloadSetup:
modified_final_response_obj: Final = redact_message_input_output_from_logging(
model_call_details=kwargs,
result=final_response_obj,
result=overlay_served_output_texts(final_response_obj, kwargs.get(SERVED_OUTPUT_TEXTS_KEY)),
)
if modified_final_response_obj is not None and isinstance(modified_final_response_obj, BaseModel):
final_response_obj = modified_final_response_obj.model_dump()
else:
final_response_obj = modified_final_response_obj
return final_response_obj
return modified_final_response_obj.model_dump()
return modified_final_response_obj
@staticmethod
def get_additional_headers(
@ -6277,6 +6329,8 @@ def _extract_response_obj_and_hidden_params(
hidden_params = getattr(init_response_obj, "_hidden_params", None)
elif isinstance(init_response_obj, dict):
response_obj = init_response_obj
elif isinstance(init_response_obj, HttpxBinaryResponseContent):
response_obj = dict(init_response_obj.logging_summary())
else:
response_obj = {}

View file

@ -67,6 +67,15 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
)
_INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"})
_BATCH_KEY_SUFFIX: Final = "_batches"
_BATCH_RATE_PREFIXES: Final = (
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
)
_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$")
_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX)
def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool:
@ -248,9 +257,80 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
return f"{base_key}_{suffix}"
def _parse_token_threshold(threshold: str) -> float:
return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1)
def _parse_above_token_threshold(key: str) -> float:
threshold_str: Final = key.split("_above_")[1].split("_tokens")[0]
return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1)
return _parse_token_threshold(key.split("_above_")[1].split("_tokens")[0])
def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool:
return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold)
@dataclass(frozen=True, slots=True)
class BatchCostRates:
input: float | None
output: float | None
cache_read: float | None
cache_creation: float | None
def _batch_rate(model_info: ModelInfo, key: str) -> float | None:
value: Final = model_info.get(key)
if isinstance(value, (int, float)):
return float(value)
if not isinstance(value, str):
return None
try:
return float(value)
except ValueError:
return None
def _batch_tier_rate(model_info: ModelInfo, tier_key: str, flat_key: str) -> float | None:
tier_rate: Final = _batch_rate(model_info, tier_key)
return _batch_rate(model_info, flat_key) if tier_rate is None else tier_rate
def _batch_tier_thresholds(model_info: ModelInfo, prefix: str) -> frozenset[str]:
return frozenset(
tier.group(2)
for key, value in model_info.items()
if value is not None and (tier := _BATCH_TIER_KEY.match(key)) is not None and tier.group(1) == prefix
)
def _crossed_batch_tier(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> str | None:
return next(
(
threshold
for threshold in sorted(
_batch_tier_thresholds(model_info, prefix), key=_parse_token_threshold, reverse=True
)
if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_token_threshold(threshold), inclusive)
),
None,
)
def _batch_rate_for_prefix(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> float | None:
flat_key: Final = f"{prefix}{_BATCH_KEY_SUFFIX}"
threshold: Final = _crossed_batch_tier(model_info, prefix, usage, inclusive)
if threshold is None:
return _batch_rate(model_info, flat_key)
return _batch_tier_rate(model_info, f"{prefix}_above_{threshold}_tokens{_BATCH_KEY_SUFFIX}", flat_key)
def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None) -> BatchCostRates:
inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider)
return BatchCostRates(
input=_batch_rate_for_prefix(model_info, "input_cost_per_token", usage, inclusive),
output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive),
cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive),
cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive),
)
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None:
@ -576,7 +656,9 @@ def _get_token_base_cost(
# so that the threshold detection loop only processes standard keys. The
# service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
threshold_keys: Final = [
k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES)
k
for k in model_info
if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES)
]
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
@ -588,7 +670,7 @@ def _get_token_base_cost(
# Handle both formats: _above_128k_tokens and _above_128_tokens
threshold_str = key.split("_above_")[1].split("_tokens")[0]
threshold = _parse_above_token_threshold(key)
if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold):
if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive):
# Prefer a service_tier-specific above-threshold key when available,
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
# ON_DEMAND_PRIORITY. Falls back to the standard key automatically

View file

@ -251,6 +251,6 @@ def update_response_metadata(
return
metadata: Final = ResponseMetadata(result)
metadata.set_hidden_params(logging_obj, model, kwargs)
metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
metadata.set_hidden_params(logging_obj, model, kwargs)
metadata.apply()

View file

@ -20,6 +20,7 @@ from litellm.litellm_core_utils.classifier_logging import without_classifier_aud
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
)
from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
from litellm.llms.vertex_ai.common_utils import (
redact_vertex_ai_metadata_from_litellm_params,
redact_vertex_ai_metadata_from_logged_object,
@ -245,6 +246,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_tool_calls_dict(choice["delta"])
elif choice.get("text") is not None:
choice["text"] = redacted_str
else:
_redact_choice_content(choice)
@ -265,6 +268,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
model_call_details["prompt"] = ""
model_call_details["input"] = ""
model_call_details.pop(SERVED_OUTPUT_TEXTS_KEY, None)
standard_logging_object: Final = model_call_details.get("standard_logging_object")
if isinstance(standard_logging_object, Mapping):
model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object)

View file

@ -0,0 +1,171 @@
"""Assistant text the caller received, per choice, so the logging payload stores the response a
post-call guardrail rewrote rather than the provider response the proxy assembled before it ran."""
from __future__ import annotations
from collections.abc import Sequence
from typing import Final, Literal
from pydantic import BaseModel, TypeAdapter, ValidationError
from litellm.types.utils import ModelResponse, ModelResponseStream
SERVED_OUTPUT_TEXTS_KEY: Final = "served_output_texts"
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
_JSON_LIST: Final = TypeAdapter(list[object])
_TEXTS: Final = TypeAdapter(tuple[str | None, ...])
ServedTexts = tuple[str | None, ...]
class _TextBlock(BaseModel):
type: str
text: str | None = None
class _AnthropicMessage(BaseModel):
type: Literal["message"]
content: list[_TextBlock]
class _ResponsesOutputItem(BaseModel):
type: str
content: list[_TextBlock] = []
class _ResponsesResponse(BaseModel):
object: Literal["response"]
output: list[_ResponsesOutputItem]
class _ChatChoices(BaseModel):
choices: list[object]
def _as_json_object(response: object) -> dict[str, object] | None:
candidate: Final = response.model_dump() if isinstance(response, BaseModel) else response
try:
return _JSON_OBJECT.validate_python(candidate)
except ValidationError:
return None
def _joined_block_texts(blocks: Sequence[_TextBlock], *, text_type: str) -> str | None:
texts: Final = tuple(block.text for block in blocks if block.type == text_type and block.text is not None)
return "".join(texts) if texts else None
def _chat_texts(response: ModelResponse) -> ServedTexts | None:
texts: Final = tuple(
choice.message.content if isinstance(choice.message.content, str) else None for choice in response.choices
)
return texts if any(text is not None for text in texts) else None
def _anthropic_message_text(response: dict[str, object]) -> str | None:
try:
message: Final = _AnthropicMessage.model_validate(response)
except ValidationError:
return None
return _joined_block_texts(message.content, text_type="text")
def _responses_api_text(response: dict[str, object]) -> str | None:
try:
parsed: Final = _ResponsesResponse.model_validate(response)
except ValidationError:
return None
texts: Final = tuple(
text
for item in parsed.output
if item.type == "message" and (text := _joined_block_texts(item.content, text_type="output_text")) is not None
)
return "".join(texts) if texts else None
def _chat_dict_texts(response: dict[str, object]) -> ServedTexts | None:
try:
_ChatChoices.model_validate(response)
return _chat_texts(ModelResponse(**response))
except (ValidationError, TypeError, ValueError):
return None
def served_output_texts(response: object) -> ServedTexts | None:
if isinstance(response, ModelResponse):
return _chat_texts(response)
mapping: Final = _as_json_object(response)
if mapping is None:
return None
chat_texts: Final = _chat_dict_texts(mapping)
if chat_texts is not None:
return chat_texts
anthropic_text: Final = _anthropic_message_text(mapping)
text: Final = anthropic_text if anthropic_text is not None else _responses_api_text(mapping)
return (text,) if text is not None else None
def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None:
if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks):
return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)))
from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_anthropic_sse_stream
if not is_anthropic_sse_stream(chunks):
return None
assembled: Final = assemble_anthropic_sse_stream(chunks)
return _chat_texts(assembled) if assembled is not None else None
def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int) -> str | None:
contents: Final = tuple(
content
for chunk in chunks
for choice in chunk.choices
if choice.index == index and isinstance(content := choice.delta.content, str)
)
return "".join(contents) if contents else None
def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None:
choice_count: Final = max((choice.index + 1 for chunk in chunks for choice in chunk.choices), default=0)
texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count))
return texts if any(text is not None for text in texts) else None
def record_served_output_texts(model_call_details: dict[str, object], texts: ServedTexts | None) -> None:
if texts is None:
return
model_call_details[SERVED_OUTPUT_TEXTS_KEY] = texts # rebind-ok: model_call_details is the shared kwargs bag
def overlay_served_output_texts(
response_obj: dict[str, object] | str | list[object] | None, served_texts: object
) -> dict[str, object] | str | list[object] | None:
if not isinstance(response_obj, dict):
return response_obj
logged: Final = _as_json_object(response_obj)
if logged is None:
return response_obj
try:
texts: Final = _TEXTS.validate_python(served_texts)
choices: Final = _JSON_LIST.validate_python(logged.get("choices"))
except ValidationError:
return response_obj
return {
**logged,
"choices": [
_choice_with_text(choice, texts[index]) if index < len(texts) else choice
for index, choice in enumerate(choices)
],
}
def _choice_with_text(choice: object, text: str | None) -> object:
choice_obj: Final = _as_json_object(choice)
if choice_obj is None or text is None:
return choice
message: Final = _as_json_object(choice_obj.get("message"))
if message is None or message.get("content") == text:
return choice
return {**choice_obj, "message": {**message, "content": text}}

View file

@ -10,7 +10,6 @@ import anyio
import anyio.lowlevel
import httpx
import tiktoken
from tokenizers import Tokenizer
from typing_extensions import ParamSpec, TypeVar
import litellm
@ -30,8 +29,10 @@ from litellm.constants import (
TOKEN_COUNTER_MAX_EXACT_CHARS,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding
from litellm.litellm_core_utils.url_utils import safe_get
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.rust_bridge.tokenizer import get_encoding
from litellm.types.llms.anthropic import (
AnthropicContentParamSource,
AnthropicContentParamSourceFileId,
@ -622,9 +623,11 @@ def _get_exact_count_function(
if model is not None or custom_tokenizer is not None:
tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model)
if tokenizer_json["type"] == "huggingface_tokenizer":
tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"]
tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"]
def count_tokens(text: str) -> int:
if isinstance(tokenizer, HuggingFaceTokenizer):
return tokenizer.count(text)
return len(tokenizer.encode_batch_fast([text])[0])
return count_tokens
@ -632,31 +635,43 @@ def _get_exact_count_function(
encoding: Final = openai_tokenizer_encoding(model)
def encode_length(text: str) -> int:
return len(encoding.encode(text, disallowed_special=()))
return _encoding_count(encoding, text)
return _get_tiktoken_count_function(encode_length)
else:
raise ValueError("Unsupported tokenizer type")
else:
default_encoding: Final = _get_default_encoding()
def encode_length(text: str) -> int:
return len(_get_default_encoding().encode(text, disallowed_special=()))
return _encoding_count(default_encoding, text)
return _get_tiktoken_count_function(encode_length)
def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding:
"""The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
def _encoding_count(encoding: Encoding, text: str) -> int:
if isinstance(encoding, OpenAIEncoding):
return encoding.count(text)
return len(encoding.encode(text, disallowed_special=()))
def openai_tokenizer_encoding(model: str) -> Encoding:
"""The encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
return get_encoding(openai_tokenizer_encoding_name(model))
def openai_tokenizer_encoding_name(model: str) -> str:
"""The tiktoken encoding name for `model`, without loading the encoding."""
from litellm.utils import print_verbose
model_to_use: Final = _fix_model_name(model)
if "gpt-4o" in model_to_use:
return tiktoken.get_encoding("o200k_base")
return "o200k_base"
try:
return tiktoken.encoding_for_model(model_to_use)
return tiktoken.encoding_name_for_model(model_to_use)
except KeyError:
print_verbose("Warning: model not found. Using cl100k_base encoding.")
return tiktoken.get_encoding("cl100k_base")
return "cl100k_base"
def uses_legacy_message_accounting(model: str) -> bool:

View file

@ -0,0 +1,402 @@
"""Python faces of the Rust text codecs.
``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors
``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by
``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog
selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``,
``enable_truncation``, ``add_tokens``) stay on the Python tokenizer.
"""
from __future__ import annotations
from collections.abc import Callable, Collection, Mapping, Sequence, Set
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from functools import partial
from pathlib import Path
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable
import tiktoken
from tokenizers import AddedToken
from tokenizers import Tokenizer as PythonHuggingFaceTokenizer
if TYPE_CHECKING:
import numpy as np
import numpy.typing as npt
from litellm.rust_bridge._native import HuggingFaceEncoding
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
SpecialTokens: TypeAlias = Literal["all"] | Collection[str]
AllowedSpecial: TypeAlias = Literal["all"] | Set[str]
HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...]
HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput]
@dataclass(frozen=True, slots=True)
class OpenAIEncoding:
"""``tiktoken.Encoding`` over the Rust tiktoken codec."""
_native: NativeTokenizer
_special_tokens: Mapping[str, int]
@staticmethod
def wrap(native: NativeTokenizer) -> OpenAIEncoding:
return OpenAIEncoding(native, MappingProxyType(native.special_tokens()))
@staticmethod
def from_tiktoken(encoding: str) -> OpenAIEncoding:
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding))
def __repr__(self) -> str:
return f"<Encoding {self.name!r}>"
@property
def name(self) -> str:
return self._native.name
@property
def max_token_value(self) -> int:
return self._native.max_token_value()
@property
def n_vocab(self) -> int:
"""For backwards compatibility. Prefer to use `enc.max_token_value + 1`."""
return self.max_token_value + 1
@property
def eot_token(self) -> int:
return self._special_tokens["<|endoftext|>"]
@property
def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type
return set(self._special_tokens)
def is_special_token(self, token: int) -> bool:
return self._native.is_special_token(token)
# ---- encoding -------------------------------------------------------------------------
def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.encode(text)
def encode(
self,
text: str,
*,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
if not allowed:
return self.encode_ordinary(text)
return self._native.encode_special(text, tuple(allowed))
def encode_to_numpy(
self,
text: str,
*,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> npt.NDArray[np.uint32]:
import numpy
return numpy.asarray(
self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special),
dtype=numpy.uint32,
)
def encode_ordinary_batch(
self, text: Sequence[str], *, num_threads: int = 8
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(self.encode_ordinary, text)
)
def encode_batch(
self,
text: Sequence[str],
*,
num_threads: int = 8,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special)
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(encode, text)
)
def encode_with_unstable(
self,
text: str,
*,
allowed_special: AllowedSpecial = frozenset(),
disallowed_special: SpecialTokens = "all",
) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type
"""The stable tokens of `text` and every completion its unstable tail could become.
Completions come back sorted; tiktoken returns them in hash order."""
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
return self._native.encode_with_unstable(text, tuple(allowed))
def encode_single_token(self, text_or_bytes: str | bytes) -> int:
"""The token of one whole piece, special tokens included. Raises `KeyError` otherwise."""
piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes
return self._native.encode_single_token(piece)
def count(self, text: str, fast: bool = False) -> int:
"""Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally."""
return self._native.count(text, fast)
# ---- decoding -------------------------------------------------------------------------
def decode_bytes(self, tokens: Sequence[int]) -> bytes:
return self._native.decode_bytes(tokens)
def decode(self, tokens: Sequence[int], errors: str = "replace") -> str:
return self.decode_bytes(tokens).decode("utf-8", errors=errors)
def decode_single_token_bytes(self, token: int) -> bytes:
return self.decode_bytes((token,))
def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
return [ # mutable-ok: [LIT002] SDK returns a list
self.decode_single_token_bytes(token) for token in tokens
]
def decode_with_offsets(
self, tokens: Sequence[int]
) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
"""The decoded text and, per token, the index of the first character holding its bytes.
Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8."""
token_bytes: Final = self.decode_tokens_bytes(tokens)
text_len = 0
offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator
for token in token_bytes:
offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0)))
text_len += sum(1 for c in token if not 0x80 <= c < 0xC0)
return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets
def decode_batch(
self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(partial(self.decode, errors=errors), batch)
)
def decode_bytes_batch(
self, batch: Sequence[Sequence[int]], *, num_threads: int = 8
) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
with ThreadPoolExecutor(num_threads) as executor:
return list( # mutable-ok: [LIT002] SDK returns a list
executor.map(self.decode_bytes, batch)
)
def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.token_byte_values()
def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]:
return (OpenAIEncoding.from_tiktoken, (self.name,))
# ---- private --------------------------------------------------------------------------
def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]:
"""tiktoken's special-token policy: which specials `text` may encode, after rejecting
any it must not contain."""
allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special)
disallowed: Final = (
frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special)
)
for token in disallowed:
if token in text:
raise ValueError(
f"Encountered text corresponding to disallowed special token {token!r}.\n"
"If you want this text to be encoded as a special token, "
f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n"
"If you want this text to be encoded as normal text, disable the check for this token "
f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n"
"To disable this check for all special tokens, pass `disallowed_special=()`.\n"
)
return allowed
@dataclass(frozen=True, slots=True)
class HuggingFaceTokenizer:
"""The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec."""
_native: NativeTokenizer
@staticmethod
def from_str(json: str) -> HuggingFaceTokenizer:
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
return HuggingFaceTokenizer(NativeTokenizer.from_json(json))
from_json = from_str
@staticmethod
def from_buffer(buffer: bytes) -> HuggingFaceTokenizer:
return HuggingFaceTokenizer.from_str(buffer.decode("utf-8"))
@staticmethod
def from_file(path: str) -> HuggingFaceTokenizer:
return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8"))
@staticmethod
def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer:
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token))
def to_str(self, pretty: bool = False) -> str:
return self._native.to_json(pretty)
def save(self, path: str, pretty: bool = True) -> None:
Path(path).write_text(self.to_str(pretty), encoding="utf-8")
@property
def name(self) -> str:
return self._native.name
# ---- vocabulary -----------------------------------------------------------------------
def token_to_id(self, token: str) -> int | None:
return self._native.token_to_id(token)
def id_to_token(self, id: int) -> str | None:
return self._native.id_to_token(id)
def get_vocab(
self, with_added_tokens: bool = True
) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.get_vocab(with_added_tokens)
def get_vocab_size(self, with_added_tokens: bool = True) -> int:
return self._native.get_vocab_size(with_added_tokens)
def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type
return { # mutable-ok: [LIT002] SDK returns a dict
token_id: AddedToken(
content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special
)
for token_id, (
content,
single_word,
lstrip,
rstrip,
normalized,
special,
) in self._native.added_tokens_decoder()
}
def num_special_tokens_to_add(self, is_pair: bool) -> int:
return self._native.num_special_tokens_to_add(is_pair)
@property
def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.padding()
@property
def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
return self._native.truncation()
@property
def encode_special_tokens(self) -> bool:
return self._native.encode_special_tokens()
# ---- encoding and decoding ------------------------------------------------------------
def encode(
self,
sequence: HuggingFaceInput,
pair: HuggingFaceInput | None = None,
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> HuggingFaceEncoding:
return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens)
def encode_batch(
self,
input: Sequence[HuggingFaceBatchInput],
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False)
def encode_batch_fast(
self,
input: Sequence[HuggingFaceBatchInput],
is_pretokenized: bool = False,
add_special_tokens: bool = True,
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True)
def _encode_batch(
self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input)
return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast)
def count(self, text: str, fast: bool = False) -> int:
"""Count with this tokenizer's configuration; `fast` uses acceleration where supported."""
return self._native.count(text, fast)
def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str:
return self._native.decode(ids, skip_special_tokens=skip_special_tokens)
def decode_batch(
self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
return [ # mutable-ok: [LIT002] SDK returns a list
self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences
]
def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]:
return (HuggingFaceTokenizer.from_str, (self.to_str(),))
def _batch_input(
item: HuggingFaceBatchInput, is_pretokenized: bool
) -> tuple[HuggingFaceInput, HuggingFaceInput | None]:
if isinstance(item, str):
return (item, None)
if is_pretokenized and all(isinstance(word, str) for word in item):
return (tuple(word for word in item if isinstance(word, str)), None)
if len(item) != 2:
raise TypeError("batch input must be a sequence or a pair of sequences")
return (item[0], item[1])
Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding
HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer
Tokenizer: TypeAlias = Encoding | HuggingFace
class _AddedToken(Protocol):
@property
def special(self) -> bool: ...
@runtime_checkable
class _AddedTokenDecoder(Protocol):
def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ...
def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]:
"""Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's
`decode(skip_special_tokens=True)` already does this itself."""
if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder):
return tokens
try:
added: Final = tokenizer.get_added_tokens_decoder()
except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding
return tokens
special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special)
return tuple(token for token in tokens if token not in special_ids)

View file

@ -23,9 +23,8 @@ from ..common_utils import (
from .streaming_iterator import A2AModelResponseIterator
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = (
@ -292,7 +291,7 @@ class A2AConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -14,9 +14,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import ImageObject, ImageResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import Choices, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse
from ...openai_like.chat.transformation import OpenAILikeChatConfig
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AmazonNovaChatConfig(OpenAILikeChatConfig):
@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -96,13 +96,13 @@ from ..common_utils import (
AnthropicModelInfo,
eager_input_streaming_flag,
process_anthropic_headers,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -1771,7 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
return tools
def _ensure_beta_header(self, headers: dict, beta_value: str) -> None:
def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None:
"""
Ensure a beta header value is present in the anthropic-beta header.
Merges with existing values instead of overriding them.
@ -1780,13 +1780,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers: Dictionary of headers to update
beta_value: The beta header value to add
"""
existing_beta: Final = headers.get("anthropic-beta")
if existing_beta is None:
headers["anthropic-beta"] = beta_value
return
existing_values: Final = [beta.strip() for beta in existing_beta.split(",")]
if beta_value not in existing_values:
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
existing_values: Final = tuple(
beta.strip()
for key, value in headers.items()
if key.lower() == "anthropic-beta"
for beta in value.split(",")
if beta.strip()
)
for key in tuple(headers):
if key.lower() == "anthropic-beta":
headers.pop(key)
headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value)))
def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None:
"""
@ -1824,7 +1828,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
)
def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict:
def update_headers_with_optional_anthropic_beta(
self, headers: dict, optional_params: dict, messages: Sequence[object] = ()
) -> dict:
"""Update headers with optional anthropic beta."""
# Skip adding beta headers for Vertex requests
@ -1833,6 +1839,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if is_vertex_request:
return headers
if requires_native_compaction_beta(self._resolved_provider, optional_params, messages):
self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
_tools: Final = optional_params.get("tools", [])
for tool in _tools:
if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value):
@ -1929,8 +1938,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
custom_llm_provider=self._resolved_provider,
)
headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
# === Tool-name sanitization (single chokepoint) ===
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
# sanitize *here* -- not in map_openai_params -- because:
@ -1977,6 +1984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
message=f"{e}\nReceived Messages={messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(
headers=headers, optional_params=optional_params, messages=anthropic_messages
)
## Auto-strip advisor blocks from history if advisor tool is absent.
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
_all_tools: Final = optional_params.get("tools") or []
@ -2688,7 +2699,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -79,6 +79,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
def requires_native_compaction_beta(
custom_llm_provider: str,
optional_params: Mapping[str, object],
messages: Sequence[object],
) -> bool:
return custom_llm_provider == "anthropic" and (
optional_params.get("compaction") is not None
or any(
isinstance(block, Mapping)
and block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and bool(block.get("signature"))
for message in messages
if isinstance(message, Mapping)
for content in (message.get("content"),)
if isinstance(content, (list, tuple))
for block in content
)
)
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.utils import supports_prompt_caching

View file

@ -0,0 +1,71 @@
from __future__ import annotations
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
from litellm.llms.compaction import CompactionProtocol
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction
_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_HEADERS: Final = TypeAdapter(dict[str, str])
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice")
def supports_native_compaction(params: Mapping[str, object]) -> bool:
from litellm.utils import get_model_info
if params.get("custom_llm_provider") not in (None, "anthropic", "openai"):
return False
model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/")
try:
return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True
except Exception:
return False
def compatible_defaults(payload: Mapping[str, object]) -> bool:
return all(payload.get(key) is None for key in _CONFLICTS)
def request_kwargs() -> Mapping[str, object]:
operation: Final[AnthropicCompaction] = {"type": "summarize"}
return MappingProxyType(
{
"compaction": operation,
"extra_headers": _HEADERS.validate_python(
MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
),
}
)
def _native_blocks(protocol: CompactionProtocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
if protocol == "messages":
return (
_OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else ()
)
choices: Final = _OBJECTS.validate_python(response.get("choices", ()))
choice: Final = choices[0] if len(choices) == 1 else _EMPTY
message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY))
fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY)
return _OBJECTS.validate_python(fields.get("compaction_blocks", ()))
def extract_summary(protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None:
blocks: Final = _native_blocks(protocol, response)
block: Final = blocks[0] if len(blocks) == 1 else _EMPTY
content: Final = block.get("content")
return (
content
if block.get("type") == "compaction"
and isinstance(block.get("signature"), str)
and block.get("signature")
and isinstance(content, str)
and content.strip()
else None
)

View file

@ -33,7 +33,7 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AnthropicTextError(BaseLLMException):
@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -2,8 +2,11 @@ import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
@ -16,6 +19,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
TOOL_NAME_HASH_LENGTH: Final = 8
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
def _optional_attr(source: object, name: str) -> object:
@ -36,6 +40,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
return signature if isinstance(signature, str) else None
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
fields: Final = _as_string_mapping(provider_specific_fields)
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
return (
tuple(
block
for raw_block in raw_blocks
if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction"
)
if isinstance(raw_blocks, (list, tuple))
else ()
)
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
{"name", "type", "input_schema", "description", "cache_control", "strict"}
)
@ -1330,7 +1348,11 @@ class LiteLLMAnthropicMessagesAdapter:
tool_name_mapping: dict[str, str] | None = None,
) -> list[dict[str, Any]]:
new_content: Final[list[dict[str, Any]]] = []
for choice in choices:
for choice, compaction_blocks in (
(choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields")))
for choice in choices
):
new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks)
# Handle thinking blocks first
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
for thinking_block in choice.message.thinking_blocks:
@ -1365,7 +1387,7 @@ class LiteLLMAnthropicMessagesAdapter:
)
# Handle text content
if choice.message.content is not None:
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
new_content.append(
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
)
@ -1545,21 +1567,35 @@ class LiteLLMAnthropicMessagesAdapter:
openai_finish_reason=openai_finish_reason
)
anthropic_finish_reason: Final = (
"refusal"
"compaction"
if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction"
else "refusal"
if refusal_text is not None and translated_finish_reason != "max_tokens"
else translated_finish_reason
)
# extract usage
usage: Final[Usage] = getattr(response, "usage")
anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
if polyfill_result is not None and polyfill_result.iterations_usage is not None:
message_iteration: Final[UsageIteration] = {
"type": "message",
"input_tokens": anthropic_usage["input_tokens"],
"output_tokens": usage.completion_tokens or 0,
}
anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration]
message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
anthropic_usage: Final[AnthropicUsage] = (
TypeAdapter(AnthropicUsage).validate_python(
MappingProxyType(
{
**message_usage,
"iterations": (
*polyfill_iterations,
UsageIteration(
type="message",
input_tokens=message_usage.get("input_tokens", 0),
output_tokens=usage.completion_tokens or 0,
),
),
}
)
)
if polyfill_iterations is not None
else message_usage
)
translated_obj: Final = AnthropicMessagesResponse(
id=response.id,

View file

@ -580,7 +580,9 @@ def anthropic_messages_handler(
)
if anthropic_messages_provider_config is None:
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
if kwargs.get("compaction") is None and _should_route_to_responses_api(
custom_llm_provider, original_model, model
):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=max_tokens,
messages=messages,

View file

@ -24,6 +24,7 @@ from ...common_utils import (
AnthropicError,
AnthropicModelInfo,
optionally_handle_anthropic_oauth,
requires_native_compaction_beta,
strip_advisor_blocks_from_messages,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"tool_choice",
"thinking",
"context_management",
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
"output_format",
"inference_geo",
"speed",
@ -637,6 +639,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
)
beta_values.update(existing_beta)
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
# Check for context management
context_management_param: Final = optional_params.get("context_management")
if context_management_param is not None:

View file

@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig
from ..common_utils import AzureOpenAIError
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LoggingClass = LiteLLMLoggingObj
else:
@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -143,7 +143,7 @@ class AzurePassthroughConfig(BasePassthroughConfig):
) -> dict:
return BaseAzureLLM._base_validate_azure_environment(
headers=headers,
litellm_params=GenericLiteLLMParams(**{**litellm_params, "api_key": api_key}),
litellm_params=GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key}),
)
@staticmethod

View file

@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
LiteLLMLoggingObj = _LiteLLMLoggingObj
@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AzureModelRouterConfig(AzureAIStudioConfig):
@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField
from litellm.utils import _add_path_to_api_base, supports_tool_choice
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AzureFoundryErrorStrings(str, enum.Enum):
@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:

View file

@ -12,9 +12,8 @@ from litellm.types.llms.openai import (
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator
from typing import TYPE_CHECKING, Union
if TYPE_CHECKING:
import tiktoken
from pydantic import BaseModel
from litellm import LiteLLMLoggingObj, ModelResponse
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.types.llms.openai import AllMessageValues
@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC):
messages: list["AllMessageValues"],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

View file

@ -21,9 +21,8 @@ from litellm.types.llms.openai import (
)
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
from litellm.types.utils import ModelResponse
from ..base_utils import (
@ -344,7 +343,7 @@ class BaseConfig(ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> "ModelResponse":

View file

@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
from litellm.types.utils import ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

View file

@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
from litellm.types.utils import EmbeddingResponse, ModelResponse
if TYPE_CHECKING:
import tiktoken
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC):
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
encoding: "tiktoken.Encoding | None",
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ModelResponse:

Some files were not shown because too many files have changed in this diff Show more