mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'origin/main' into litellm_spend-logs-end-date
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> # Conflicts: # tests/e2e/proxy_client.py
This commit is contained in:
commit
0d5ccb6d5e
317 changed files with 16268 additions and 2249 deletions
10
.github/actions/cache-cargo-build/action.yml
vendored
10
.github/actions/cache-cargo-build/action.yml
vendored
|
|
@ -15,6 +15,12 @@ description: >-
|
|||
cache the same directory for different workloads, and a shared key would let
|
||||
whichever ran first deny the others a save.
|
||||
|
||||
inputs:
|
||||
profile:
|
||||
description: "Cargo profile the build uses (dev or release)"
|
||||
required: false
|
||||
default: "dev"
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
|
|
@ -25,6 +31,6 @@ runs:
|
|||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
litellm-rust/target
|
||||
key: ${{ runner.os }}-maturin-dev-${{ hashFiles('litellm-rust/Cargo.lock') }}
|
||||
key: ${{ runner.os }}-maturin-${{ inputs.profile }}-${{ hashFiles('litellm-rust/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maturin-dev-
|
||||
${{ runner.os }}-maturin-${{ inputs.profile }}-
|
||||
|
|
|
|||
14
.github/e2e-stack/secrets_to_env.py
vendored
14
.github/e2e-stack/secrets_to_env.py
vendored
|
|
@ -9,6 +9,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
secrets_adapter: Final[TypeAdapter[dict[str, str]]] = TypeAdapter(dict[str, str])
|
||||
ENV_NAME: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
MIN_MASKED_LENGTH: Final = 8
|
||||
ACTIONS_RUNNER_FLAG: Final = "GITHUB_ACTIONS"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
|
|
@ -30,10 +31,15 @@ def main() -> int:
|
|||
f"these names or values cannot be represented in both bash and dotenv: {' '.join(sorted(unusable))}\n"
|
||||
)
|
||||
return 1
|
||||
for value in secrets.values():
|
||||
if len(value) >= MIN_MASKED_LENGTH:
|
||||
_ = sys.stdout.write(f"::add-mask::{value.replace('%', '%25')}\n")
|
||||
sys.stdout.flush()
|
||||
if os.environ.get(ACTIONS_RUNNER_FLAG) == "true":
|
||||
_ = sys.stdout.write(
|
||||
"".join(
|
||||
f"::add-mask::{value.replace('%', '%25')}\n"
|
||||
for value in secrets.values()
|
||||
if len(value) >= MIN_MASKED_LENGTH
|
||||
)
|
||||
)
|
||||
sys.stdout.flush()
|
||||
lines: Final = tuple(f"{key}='{value}'" for key, value in secrets.items() if value)
|
||||
try:
|
||||
with os.fdopen(os.open(env_path, os.O_WRONLY | os.O_APPEND | os.O_CREAT | os.O_NOFOLLOW, 0o600), "w") as handle:
|
||||
|
|
|
|||
1
.github/e2e-stack/select_tests.py
vendored
1
.github/e2e-stack/select_tests.py
vendored
|
|
@ -9,6 +9,7 @@ UNSUPPORTED: Final = re.compile(
|
|||
r"|^tests/e2e/llm_translation/realtime/test_realtime_pipecat_audio_e2e\.py$"
|
||||
r"|^tests/e2e/batches/test_managed_files_enforcement_e2e\.py$"
|
||||
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
|
||||
)
|
||||
HARNESS: Final = re.compile(
|
||||
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
|
||||
|
|
|
|||
14
.github/scripts/verify_linux_native_wheel.py
vendored
14
.github/scripts/verify_linux_native_wheel.py
vendored
|
|
@ -134,7 +134,16 @@ def main(
|
|||
uncompressed_wheel_size: Final = sum(member.file_size for member in wheel_members)
|
||||
native_path: Final = wheel.parent / "native" / Path(native_member.filename).name
|
||||
native_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
native_path.write_bytes(archive.read(native_member))
|
||||
native_bytes: Final = archive.read(native_member)
|
||||
native_path.write_bytes(native_bytes)
|
||||
duplicated_vocabularies: Final = tuple(
|
||||
member.filename
|
||||
for member in wheel_members
|
||||
if member.filename.startswith("litellm/litellm_core_utils/tokenizers/")
|
||||
and re.fullmatch(r"[0-9a-f]{40}", PurePosixPath(member.filename).name)
|
||||
and member.file_size > 0
|
||||
and archive.read(member) in native_bytes
|
||||
)
|
||||
|
||||
wheel_metadata_tags_match: Final = (
|
||||
len(wheel_metadata_tags) == len(expanded_filename_tags)
|
||||
|
|
@ -205,7 +214,7 @@ def main(
|
|||
native_module: Final = load_native_module(native_path)
|
||||
native_module_loads: Final = native_module is not None
|
||||
panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
|
||||
native_size_limit: Final = 40_000_000
|
||||
native_size_limit: Final = 35_000_000
|
||||
native_size_within_limit: Final = native_member.file_size <= native_size_limit
|
||||
validations: Final = (
|
||||
(f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
|
||||
|
|
@ -223,6 +232,7 @@ def main(
|
|||
("Native module loads", native_module_loads),
|
||||
("Production module omits the panic test hook", panic_test_hook_absent),
|
||||
(f"Native extension does not exceed {native_size_limit / 1_000_000:.0f} MB", native_size_within_limit),
|
||||
("Tokenizer vocabularies are not duplicated in the native extension", not duplicated_vocabularies),
|
||||
("Wheel contents are valid", not unexpected_members),
|
||||
)
|
||||
|
||||
|
|
|
|||
36
.github/workflows/codspeed.yml
vendored
36
.github/workflows/codspeed.yml
vendored
|
|
@ -13,6 +13,7 @@ on:
|
|||
- ".github/workflows/codspeed.yml"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/actions/cache-cargo-build/**"
|
||||
- ".github/scripts/uv_sync_with_retries.sh"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
|
@ -25,6 +26,7 @@ on:
|
|||
- ".github/workflows/codspeed.yml"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/actions/cache-cargo-build/**"
|
||||
- ".github/scripts/uv_sync_with_retries.sh"
|
||||
# Allow CodSpeed to trigger backtest performance analysis
|
||||
# in order to generate initial data
|
||||
workflow_dispatch:
|
||||
|
|
@ -59,19 +61,27 @@ jobs:
|
|||
|
||||
- name: Cache the Rust build
|
||||
uses: ./.github/actions/cache-cargo-build
|
||||
with:
|
||||
profile: release
|
||||
|
||||
# Build the wheel and resolve every dependency outside the CodSpeed
|
||||
# runner: the same maturin build took 42 minutes inside `codspeed run`
|
||||
# versus under 3 minutes as a plain step (LIT-6183)
|
||||
- name: Build environment
|
||||
- name: Build the release wheel
|
||||
run: uv build --wheel --out-dir dist
|
||||
|
||||
- name: Install the wheel into the benchmark environment
|
||||
run: |
|
||||
UV_PROJECT_ENVIRONMENT="${RUNNER_TEMP}/benchmark-venv" .github/scripts/uv_sync_with_retries.sh --frozen --no-default-groups --group benchmarks --no-install-project --python 3.12
|
||||
uv pip install --python "${RUNNER_TEMP}/benchmark-venv/bin/python" --no-deps dist/*.whl
|
||||
|
||||
- name: Collect benchmarks
|
||||
env:
|
||||
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
|
||||
LITELLM_REQUIRE_INSTALLED_WHEEL: "1"
|
||||
run: >
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
||||
uv run --frozen --no-default-groups
|
||||
--with pytest==8.3.5
|
||||
--with pytest-codspeed==4.3.0
|
||||
--with "mcp>=2.2.0,<3.0"
|
||||
--with "a2a-sdk>=1.1.0,<2.0"
|
||||
pytest
|
||||
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
||||
--import-mode=importlib
|
||||
-p pytest_codspeed.plugin
|
||||
tests/benchmarks/
|
||||
--codspeed
|
||||
|
|
@ -82,13 +92,9 @@ jobs:
|
|||
with:
|
||||
mode: simulation
|
||||
run: >
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
|
||||
uv run --frozen --no-default-groups
|
||||
--with pytest==8.3.5
|
||||
--with pytest-codspeed==4.3.0
|
||||
--with "mcp>=2.2.0,<3.0"
|
||||
--with "a2a-sdk>=1.1.0,<2.0"
|
||||
pytest
|
||||
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 LITELLM_REQUIRE_INSTALLED_WHEEL=1
|
||||
"${RUNNER_TEMP}/benchmark-venv/bin/python" -I -m pytest
|
||||
--import-mode=importlib
|
||||
-p pytest_codspeed.plugin
|
||||
tests/benchmarks/
|
||||
--codspeed
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
|||
--extra saml \
|
||||
--python python3.13
|
||||
|
||||
RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
|
||||
|
||||
RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
|
||||
npm_config_cache=/root/.npm \
|
||||
prisma generate --schema=./schema.prisma
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
"""Guard the cost map on pull requests.
|
||||
|
||||
Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
|
||||
and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
|
||||
litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
|
||||
Every pull request whose diff against its merge base touches one of the three cost map files gets the file
|
||||
checks: the files parse, the backup copy matches the root file, and the JSON schema is in sync and validates the
|
||||
map. A pull request that leaves all three untouched skips them, since merging it keeps the base branch's copies
|
||||
and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot
|
||||
(branches named litellm_cost_map_sync_*) always get the file checks and additionally may only touch those three
|
||||
files and may only add or update models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -108,20 +111,37 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
)
|
||||
|
||||
|
||||
def touches_cost_map(changed_files: Sequence[str]) -> bool:
|
||||
return any(path in GUARDED_PATHS for path in changed_files)
|
||||
|
||||
|
||||
def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
|
||||
if bot:
|
||||
return "bot contract enforced"
|
||||
return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
|
||||
|
||||
|
||||
def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
|
||||
if not bot and not touches_cost_map(changed_files):
|
||||
return ()
|
||||
head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
|
||||
if isinstance(head_map, str):
|
||||
return (head_map,)
|
||||
return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
|
||||
|
||||
|
||||
def _git(*args: str) -> str:
|
||||
def _git(*args: str) -> str | None:
|
||||
result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
|
||||
return result.stdout if result.returncode == 0 else ""
|
||||
return result.stdout if result.returncode == 0 else None
|
||||
|
||||
|
||||
def snapshot(revision: str) -> Snapshot:
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
|
||||
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
|
||||
|
||||
|
||||
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
|
||||
diff: Final = _git("diff", "--name-only", "--no-renames", base, head)
|
||||
return None if diff is None else tuple(diff.splitlines())
|
||||
|
||||
|
||||
def main(argv: Sequence[str]) -> int:
|
||||
|
|
@ -131,9 +151,12 @@ def main(argv: Sequence[str]) -> int:
|
|||
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
|
||||
args: Final = parser.parse_args(argv)
|
||||
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
|
||||
changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
|
||||
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
|
||||
contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
|
||||
changed: Final = changed_files(args.base, args.head)
|
||||
if changed is None:
|
||||
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
|
||||
return 1
|
||||
failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
|
||||
contract: Final = contract_for(bot, changed)
|
||||
if failures:
|
||||
print(f"cost map guard failed ({contract}):")
|
||||
print("\n".join(f"- {failure}" for failure in failures))
|
||||
|
|
|
|||
5
litellm-rust/Cargo.lock
generated
5
litellm-rust/Cargo.lock
generated
|
|
@ -3065,6 +3065,7 @@ dependencies = [
|
|||
"serde_json",
|
||||
"serde_with",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tokio-tungstenite",
|
||||
"url",
|
||||
|
|
@ -3253,6 +3254,7 @@ dependencies = [
|
|||
name = "litellm-token-counter-huggingface"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokenizers",
|
||||
]
|
||||
|
|
@ -3261,6 +3263,9 @@ dependencies = [
|
|||
name = "litellm-token-counter-tiktoken"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"once_cell",
|
||||
"rustc-hash",
|
||||
"thiserror 2.0.19",
|
||||
"tiktoken-rs",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ veil = "0.3.0"
|
|||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = "thin"
|
||||
lto = "fat"
|
||||
codegen-units = 1
|
||||
panic = "unwind"
|
||||
debug = false
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ pyo3::create_exception!(
|
|||
|
||||
static FORK_GATE: ForkGate = ForkGate::new();
|
||||
|
||||
/// Whether this process has started the Tokio runtime.
|
||||
/// Whether this process has entered process-bound native execution.
|
||||
pub fn runtime_started() -> bool {
|
||||
FORK_GATE.started(std::process::id())
|
||||
}
|
||||
|
|
@ -40,9 +40,8 @@ pub fn reserve_process_for_forking() -> Result<(), RuntimeAlreadyStarted> {
|
|||
FORK_GATE.reserve(std::process::id())
|
||||
}
|
||||
|
||||
/// The only door to the Tokio runtime: every route reaches it through this module, which is
|
||||
/// what lets the gate speak for the whole extension. `clippy.toml` disallows going around it.
|
||||
fn enter_runtime() -> PyResult<()> {
|
||||
/// Claims process-bound native state before runtime startup or tokenizer execution.
|
||||
pub fn enter_native() -> PyResult<()> {
|
||||
FORK_GATE
|
||||
.enter(std::process::id())
|
||||
.map_err(|refused| match refused {
|
||||
|
|
@ -60,7 +59,7 @@ fn enter_runtime() -> PyResult<()> {
|
|||
|
||||
#[expect(clippy::disallowed_methods, reason = "this is the gated door")]
|
||||
fn runtime() -> PyResult<&'static Runtime> {
|
||||
enter_runtime()?;
|
||||
enter_native()?;
|
||||
Ok(pyo3_async_runtimes::tokio::get_runtime())
|
||||
}
|
||||
|
||||
|
|
@ -70,7 +69,7 @@ where
|
|||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
T: for<'py> IntoPyObject<'py> + Send + 'static,
|
||||
{
|
||||
enter_runtime()?;
|
||||
enter_native()?;
|
||||
pyo3_async_runtimes::tokio::future_into_py(py, future)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ pub use argument::lookup;
|
|||
pub use callable::wrap_failure;
|
||||
pub use driver::run_call;
|
||||
pub use execution::{
|
||||
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, poll_async_value,
|
||||
ForkedAfterNativeRuntimeStarted, ProcessReservedForForking, enter_native, poll_async_value,
|
||||
reserve_process_for_forking, run_async, run_async_value, run_sync, run_sync_value,
|
||||
runtime_started,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ name = "_native"
|
|||
crate-type = ["cdylib"]
|
||||
|
||||
[features]
|
||||
default = ["abi3", "fast"]
|
||||
default = ["abi3", "fast", "huggingface", "tiktoken"]
|
||||
abi3 = ["pyo3/abi3-py310"]
|
||||
extension-module = ["pyo3/extension-module"]
|
||||
panic-test = []
|
||||
|
|
@ -52,8 +52,9 @@ pyo3-async-runtimes.workspace = true
|
|||
reqwest.workspace = true
|
||||
redis = { version = "1.7.0", features = ["tls-rustls"] }
|
||||
serde_json.workspace = true
|
||||
url.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["rt", "sync"] }
|
||||
url.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-secrets-aws.workspace = true
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ mod routes;
|
|||
reason = "secret-manager foundations await rollout activation"
|
||||
)]
|
||||
mod secrets;
|
||||
mod token_counter;
|
||||
mod tokenizer;
|
||||
|
||||
#[pymodule(gil_used = true)]
|
||||
mod _native {
|
||||
|
|
@ -37,7 +37,12 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::routes::responses::ResponsesWebSocketConnection;
|
||||
#[pymodule_export]
|
||||
use crate::token_counter::TokenCounter;
|
||||
use crate::routes::token_counter::TokenCounter;
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pymodule_export]
|
||||
use crate::tokenizer::HuggingFaceEncoding;
|
||||
#[pymodule_export]
|
||||
use crate::tokenizer::Tokenizer;
|
||||
#[pymodule_export]
|
||||
use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking};
|
||||
use pyo3::{prelude::*, types::PyModule};
|
||||
|
|
@ -83,10 +88,13 @@ mod tests {
|
|||
"achat_completions",
|
||||
"ResponsesWebSocketConnection",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"gil_stats",
|
||||
"process_state_started",
|
||||
"reserve_process_for_forking",
|
||||
];
|
||||
#[cfg(feature = "huggingface")]
|
||||
expected.push("HuggingFaceEncoding");
|
||||
expected.sort_unstable();
|
||||
|
||||
let mut public_names: Vec<String> = native_module(py)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ pub(crate) mod chat_completions;
|
|||
pub(crate) mod messages;
|
||||
pub(crate) mod ocr;
|
||||
pub(crate) mod responses;
|
||||
pub(crate) mod token_counter;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
|
|
|||
|
|
@ -131,7 +131,7 @@ impl RouteHost for OcrRouteHost {
|
|||
|
||||
fn classify(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
|
||||
if let Error::Secret(source) = &error
|
||||
&& let Some(original) = crate::secrets::callback::python_error(py, source)
|
||||
&& let Some(original) = crate::secrets::python_error(py, source)
|
||||
{
|
||||
return Ok(original);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
use std::sync::Arc;
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
use litellm_host_python::{enter_native, run_async};
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
types::PyAny,
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::tokenizer::Tokenizer;
|
||||
|
||||
/// Counts the input tokens of a raw request body off the Python event loop with
|
||||
/// the GIL released. Python owns which requests get here and what to do with
|
||||
/// the count. At most one encode per core runs at a time; the rest wait in the
|
||||
/// async task, where a cancelled Python awaiter drops them before any blocking
|
||||
/// work is scheduled.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct TokenCounter {
|
||||
inner: Arc<CoreTokenCounter>,
|
||||
encode_slots: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl TokenCounter {
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (tokenizer, fast = false))]
|
||||
fn from_tokenizer(py: Python<'_>, tokenizer: &Tokenizer, fast: bool) -> PyResult<Self> {
|
||||
enter_native()?;
|
||||
let inner = CoreTokenCounter::new(tokenizer.counter(py, fast));
|
||||
Ok(Self {
|
||||
inner: Arc::new(inner),
|
||||
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
|
||||
})
|
||||
}
|
||||
|
||||
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
|
||||
let counter = Arc::clone(&self.inner);
|
||||
let encode_slots = Arc::clone(&self.encode_slots);
|
||||
let body = body.to_vec();
|
||||
run_async(
|
||||
py,
|
||||
async move {
|
||||
let _slot = encode_slots
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?;
|
||||
tokio::task::spawn_blocking(move || count_body(&counter, &body))
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?
|
||||
},
|
||||
token_count_error_to_pyerr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_parallelism() -> usize {
|
||||
available_parallelism().map_or(1, NonZero::get)
|
||||
}
|
||||
|
||||
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
|
||||
let request = CountableRequest::parse(body)?;
|
||||
counter.count_request(&request)
|
||||
}
|
||||
|
||||
pub(crate) fn token_count_error_to_pyerr(error: Error) -> PyErr {
|
||||
let message = error.to_string();
|
||||
match error {
|
||||
Error::Load(_)
|
||||
| Error::Ranks(_)
|
||||
| Error::UnicodeClasses
|
||||
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
|
||||
Error::RequestParse(_)
|
||||
| Error::MissingInput
|
||||
| Error::FloatText
|
||||
| Error::ContentBlock
|
||||
| Error::ArrayItems
|
||||
| Error::JsonSerialization(_)
|
||||
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
|
||||
Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
|
||||
}
|
||||
}
|
||||
|
|
@ -1,45 +1,20 @@
|
|||
use std::{fmt, future::Future, pin::Pin};
|
||||
use std::{future::Future, pin::Pin};
|
||||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secrets::{
|
||||
Error, ExternalSecretManager, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
|
||||
};
|
||||
use pyo3::{exceptions::PyBaseException, prelude::*, types::PyDict};
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
use super::error::external_error;
|
||||
|
||||
const HANDLER_MODULE: &str = "litellm.secret_managers.secret_manager_handler";
|
||||
|
||||
struct PythonSecretError(Py<PyBaseException>);
|
||||
|
||||
impl fmt::Debug for PythonSecretError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("PythonSecretError")
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PythonSecretError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("Python secret manager failed")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PythonSecretError {}
|
||||
|
||||
pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option<PyErr> {
|
||||
let Error::ExternalManager(source) = error else {
|
||||
return None;
|
||||
};
|
||||
source
|
||||
.downcast_ref::<PythonSecretError>()
|
||||
.map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any()))
|
||||
}
|
||||
|
||||
/// A secret manager whose reads execute in Python: a custom manager, a legacy compatible
|
||||
/// client, or a manually assigned SDK client.
|
||||
pub(crate) struct PythonSecretManager {
|
||||
client: Py<PyAny>,
|
||||
system: Option<KeyManagementSystem>,
|
||||
/// The `key_manager` name Python's handler dispatches on.
|
||||
key_manager: &'static str,
|
||||
settings: Option<Py<PyAny>>,
|
||||
}
|
||||
|
||||
|
|
@ -52,7 +27,6 @@ impl PythonSecretManager {
|
|||
Self {
|
||||
client,
|
||||
system,
|
||||
key_manager: system.map_or("local", python_name),
|
||||
settings,
|
||||
}
|
||||
}
|
||||
|
|
@ -78,14 +52,12 @@ impl PythonSecretManager {
|
|||
}
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("client", client)?;
|
||||
kwargs.set_item("key_manager", self.key_manager)?;
|
||||
kwargs.set_item("key_manager", self.system.map_or("local", python_name))?;
|
||||
kwargs.set_item("secret_name", name)?;
|
||||
kwargs.set_item(
|
||||
"key_management_settings",
|
||||
self.settings
|
||||
.as_ref()
|
||||
.map_or_else(|| py.None(), |settings| settings.clone_ref(py)),
|
||||
)?;
|
||||
match &self.settings {
|
||||
Some(settings) => kwargs.set_item("key_management_settings", settings.bind(py))?,
|
||||
None => kwargs.set_item("key_management_settings", py.None())?,
|
||||
}
|
||||
py.import(HANDLER_MODULE)?
|
||||
.getattr("get_secret_from_manager")?
|
||||
.call((), Some(&kwargs))?
|
||||
|
|
@ -123,9 +95,7 @@ impl ExternalSecretManager for PythonSecretManager {
|
|||
Python::attach(|py| {
|
||||
self.read(py, name)
|
||||
.map(|value| value.map(SecretValue::new).map(Secret::String))
|
||||
.map_err(|error| {
|
||||
Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py))))
|
||||
})
|
||||
.map_err(|error| external_error(py, error))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
@ -140,19 +110,27 @@ mod tests {
|
|||
SecretManagerState, SecretResolver,
|
||||
};
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
use rstest::rstest;
|
||||
|
||||
use super::{HANDLER_MODULE, PythonSecretManager, python_error, python_name};
|
||||
use super::{HANDLER_MODULE, PythonSecretManager, python_name};
|
||||
use crate::secrets::python_error;
|
||||
|
||||
#[rstest]
|
||||
#[case::value_error("ValueError", None)]
|
||||
#[case::value_error_with_fallback("ValueError", Some("environment-key"))]
|
||||
#[case::cancelled("asyncio.CancelledError", None)]
|
||||
#[case::cancelled_with_fallback("asyncio.CancelledError", Some("environment-key"))]
|
||||
#[tokio::test]
|
||||
async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback() {
|
||||
async fn callback_failures_preserve_python_exceptions_even_with_environment_fallback(
|
||||
#[case] failure_type: &str,
|
||||
#[case] fallback: Option<&'static str>,
|
||||
) {
|
||||
Python::initialize();
|
||||
for failure_type in ["ValueError", "asyncio.CancelledError"] {
|
||||
for fallback in [None, Some("environment-key")] {
|
||||
let (reader, locals) = Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("failure_type", failure_type).unwrap();
|
||||
py.run(
|
||||
c"
|
||||
let (reader, locals) = Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("failure_type", failure_type).unwrap();
|
||||
py.run(
|
||||
c"
|
||||
import asyncio
|
||||
failure = eval(failure_type)('secret manager failed')
|
||||
cause = RuntimeError('original cause')
|
||||
|
|
@ -164,48 +142,46 @@ class Manager:
|
|||
raise failure
|
||||
manager = Manager()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let reader = PythonSecretManager::new(
|
||||
locals.get_item("manager").unwrap().unwrap().unbind(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
(reader, locals.unbind())
|
||||
});
|
||||
let resolver = SecretResolver::new(
|
||||
Arc::new(SecretManagerState::new(
|
||||
SecretManager::External(Arc::new(reader)),
|
||||
KeyManagementSettings::default(),
|
||||
)),
|
||||
Arc::new(move |_: &str| fallback.map(str::to_owned)),
|
||||
OidcResolver::default(),
|
||||
)
|
||||
.with_failure_policy(FailurePolicy::EnvironmentFallback);
|
||||
let error = resolver.get_secret("API_KEY", None).await.unwrap_err();
|
||||
Python::attach(|py| {
|
||||
let original = python_error(py, &error).unwrap();
|
||||
let locals = locals.bind(py);
|
||||
assert!(
|
||||
original
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] {
|
||||
assert!(
|
||||
original
|
||||
.value(py)
|
||||
.getattr(attribute)
|
||||
.unwrap()
|
||||
.is(locals.get_item(name).unwrap().unwrap())
|
||||
);
|
||||
}
|
||||
assert!(original.traceback(py).is_some());
|
||||
});
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let reader = PythonSecretManager::new(
|
||||
locals.get_item("manager").unwrap().unwrap().unbind(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
(reader, locals.unbind())
|
||||
});
|
||||
let resolver = SecretResolver::new(
|
||||
Arc::new(SecretManagerState::new(
|
||||
SecretManager::External(Arc::new(reader)),
|
||||
KeyManagementSettings::default(),
|
||||
)),
|
||||
Arc::new(move |_: &str| fallback.map(str::to_owned)),
|
||||
OidcResolver::default(),
|
||||
)
|
||||
.with_failure_policy(FailurePolicy::EnvironmentFallback);
|
||||
let error = resolver.get_secret("API_KEY", None).await.unwrap_err();
|
||||
Python::attach(|py| {
|
||||
let original = python_error(py, &error).unwrap();
|
||||
let locals = locals.bind(py);
|
||||
assert!(
|
||||
original
|
||||
.value(py)
|
||||
.is(locals.get_item("failure").unwrap().unwrap())
|
||||
);
|
||||
for (attribute, name) in [("__cause__", "cause"), ("__context__", "context")] {
|
||||
assert!(
|
||||
original
|
||||
.value(py)
|
||||
.getattr(attribute)
|
||||
.unwrap()
|
||||
.is(locals.get_item(name).unwrap().unwrap())
|
||||
);
|
||||
}
|
||||
}
|
||||
assert!(original.traceback(py).is_some());
|
||||
});
|
||||
}
|
||||
|
||||
/// Installs a fake `get_secret_from_manager` that records its kwargs, runs `body`, and
|
||||
|
|
@ -245,47 +221,57 @@ for name in installed:
|
|||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn python_names_round_trip_through_serde() {
|
||||
for system in [
|
||||
KeyManagementSystem::GoogleKms,
|
||||
KeyManagementSystem::AzureKeyVault,
|
||||
KeyManagementSystem::AwsSecretManager,
|
||||
KeyManagementSystem::GoogleSecretManager,
|
||||
KeyManagementSystem::HashicorpVault,
|
||||
KeyManagementSystem::Cyberark,
|
||||
KeyManagementSystem::Local,
|
||||
KeyManagementSystem::AwsKms,
|
||||
KeyManagementSystem::Custom,
|
||||
] {
|
||||
assert_eq!(
|
||||
serde_json::to_value(system).unwrap(),
|
||||
serde_json::Value::String(python_name(system).to_owned())
|
||||
);
|
||||
}
|
||||
#[rstest]
|
||||
#[case::google_kms(KeyManagementSystem::GoogleKms)]
|
||||
#[case::azure_key_vault(KeyManagementSystem::AzureKeyVault)]
|
||||
#[case::aws_secret_manager(KeyManagementSystem::AwsSecretManager)]
|
||||
#[case::google_secret_manager(KeyManagementSystem::GoogleSecretManager)]
|
||||
#[case::hashicorp_vault(KeyManagementSystem::HashicorpVault)]
|
||||
#[case::cyberark(KeyManagementSystem::Cyberark)]
|
||||
#[case::local(KeyManagementSystem::Local)]
|
||||
#[case::aws_kms(KeyManagementSystem::AwsKms)]
|
||||
#[case::custom(KeyManagementSystem::Custom)]
|
||||
fn python_names_round_trip_through_serde(#[case] system: KeyManagementSystem) {
|
||||
assert_eq!(
|
||||
serde_json::to_value(system).unwrap(),
|
||||
serde_json::Value::String(python_name(system).to_owned())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_readers_without_a_system_are_called_directly() {
|
||||
#[rstest]
|
||||
#[case::legacy(None, false)]
|
||||
#[case::custom(Some(KeyManagementSystem::Custom), true)]
|
||||
fn direct_readers_receive_compatible_kwargs(
|
||||
#[case] system: Option<KeyManagementSystem>,
|
||||
#[case] expects_optional_params: bool,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"
|
||||
class Settings:
|
||||
def model_dump(self):
|
||||
return {'scope': 'custom'}
|
||||
class Manager:
|
||||
def __init__(self):
|
||||
self.names = []
|
||||
self.optional_params = []
|
||||
def sync_read_secret(self, secret_name, optional_params=None, timeout=None):
|
||||
self.names.append(secret_name)
|
||||
self.optional_params.append(optional_params)
|
||||
return 'direct-' + secret_name
|
||||
manager = Manager()
|
||||
settings = Settings()
|
||||
",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let manager = locals.get_item("manager").unwrap().unwrap();
|
||||
let reader = PythonSecretManager::new(manager.clone().unbind(), None, None);
|
||||
let settings = expects_optional_params
|
||||
.then(|| locals.get_item("settings").unwrap().unwrap().unbind());
|
||||
let reader = PythonSecretManager::new(manager.clone().unbind(), system, settings);
|
||||
assert_eq!(
|
||||
reader.read(py, "API_KEY").unwrap().as_deref(),
|
||||
Some("direct-API_KEY")
|
||||
|
|
@ -298,6 +284,23 @@ manager = Manager()
|
|||
.unwrap(),
|
||||
["API_KEY"]
|
||||
);
|
||||
let optional_params = manager
|
||||
.getattr("optional_params")
|
||||
.unwrap()
|
||||
.get_item(0)
|
||||
.unwrap();
|
||||
if expects_optional_params {
|
||||
assert_eq!(
|
||||
optional_params
|
||||
.get_item("scope")
|
||||
.unwrap()
|
||||
.extract::<String>()
|
||||
.unwrap(),
|
||||
"custom"
|
||||
);
|
||||
} else {
|
||||
assert!(optional_params.is_none());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
27
litellm-rust/crates/python-bridge/src/secrets/error.rs
Normal file
27
litellm-rust/crates/python-bridge/src/secrets/error.rs
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
use std::fmt;
|
||||
|
||||
use litellm_secrets::Error;
|
||||
use pyo3::{exceptions::PyBaseException, prelude::*};
|
||||
|
||||
#[derive(thiserror::Error)]
|
||||
#[error("Python secret manager failed")]
|
||||
struct PythonSecretError(Py<PyBaseException>);
|
||||
|
||||
impl fmt::Debug for PythonSecretError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("PythonSecretError")
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn external_error(py: Python<'_>, error: PyErr) -> Error {
|
||||
Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py))))
|
||||
}
|
||||
|
||||
pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option<PyErr> {
|
||||
let Error::ExternalManager(source) = error else {
|
||||
return None;
|
||||
};
|
||||
source
|
||||
.downcast_ref::<PythonSecretError>()
|
||||
.map(|error| PyErr::from_value(error.0.clone_ref(py).into_bound(py).into_any()))
|
||||
}
|
||||
|
|
@ -1,3 +1,6 @@
|
|||
pub(crate) mod callback;
|
||||
pub(crate) mod config;
|
||||
mod error;
|
||||
pub(crate) mod resolved;
|
||||
|
||||
pub(crate) use error::python_error;
|
||||
|
|
|
|||
|
|
@ -1,158 +0,0 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_host_python::run_async;
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
types::PyAny,
|
||||
};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
/// Counts the input tokens of a raw request body off the Python event loop with
|
||||
/// the GIL released. Python owns which requests get here and what to do with
|
||||
/// the count. At most one encode per core runs at a time; the rest wait in the
|
||||
/// async task, where a cancelled Python awaiter drops them before any blocking
|
||||
/// work is scheduled.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct TokenCounter {
|
||||
inner: Arc<CoreTokenCounter>,
|
||||
encode_slots: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl TokenCounter {
|
||||
#[new]
|
||||
fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json))
|
||||
}
|
||||
#[cfg(all(not(feature = "fast"), feature = "huggingface"))]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
|
||||
}
|
||||
#[cfg(not(any(feature = "fast", feature = "huggingface")))]
|
||||
{
|
||||
let _ = (py, tokenizer_json);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast or huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
{
|
||||
let _ = (py, rank_file);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "fast")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
{
|
||||
let _ = (py, rank_file);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the fast feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
{
|
||||
Self::load(py, || CoreTokenCounter::from_tiktoken(encoding))
|
||||
}
|
||||
#[cfg(not(feature = "tiktoken"))]
|
||||
{
|
||||
let _ = (py, encoding);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the tiktoken feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
|
||||
let counter = Arc::clone(&self.inner);
|
||||
let encode_slots = Arc::clone(&self.encode_slots);
|
||||
let body = body.to_vec();
|
||||
run_async(
|
||||
py,
|
||||
async move {
|
||||
let _slot = encode_slots
|
||||
.acquire_owned()
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?;
|
||||
tokio::task::spawn_blocking(move || count_body(&counter, &body))
|
||||
.await
|
||||
.map_err(|error| Error::Task(error.to_string()))?
|
||||
},
|
||||
token_count_error_to_pyerr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenCounter {
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
fn load(
|
||||
py: Python<'_>,
|
||||
load: impl FnOnce() -> Result<CoreTokenCounter, Error> + Send,
|
||||
) -> PyResult<Self> {
|
||||
let inner = release_gil(py, load).map_err(token_count_error_to_pyerr)?;
|
||||
Ok(Self {
|
||||
inner: Arc::new(inner),
|
||||
encode_slots: Arc::new(Semaphore::new(encode_parallelism())),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
|
||||
fn encode_parallelism() -> usize {
|
||||
available_parallelism().map_or(1, NonZero::get)
|
||||
}
|
||||
|
||||
fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount, Error> {
|
||||
let request = CountableRequest::parse(body)?;
|
||||
counter.count_request(&request)
|
||||
}
|
||||
|
||||
fn token_count_error_to_pyerr(error: Error) -> PyErr {
|
||||
let message = error.to_string();
|
||||
match error {
|
||||
Error::Load(_)
|
||||
| Error::Ranks(_)
|
||||
| Error::UnicodeClasses
|
||||
| Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
|
||||
Error::RequestParse(_)
|
||||
| Error::MissingInput
|
||||
| Error::FloatText
|
||||
| Error::ContentBlock
|
||||
| Error::ArrayItems
|
||||
| Error::JsonSerialization(_)
|
||||
| Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message),
|
||||
Error::Encode(_) | Error::Task(_) => PyRuntimeError::new_err(message),
|
||||
}
|
||||
}
|
||||
713
litellm-rust/crates/python-bridge/src/tokenizer.rs
Normal file
713
litellm-rust/crates/python-bridge/src/tokenizer.rs
Normal file
|
|
@ -0,0 +1,713 @@
|
|||
//! The Python face of the text codecs: one `Tokenizer` class over the tiktoken and Hugging
|
||||
//! Face backends, carrying the read-only surface of `tiktoken.Encoding` and
|
||||
//! `tokenizers.Tokenizer` that `litellm/litellm_core_utils/tokenizer.py` wraps.
|
||||
use std::borrow::Cow;
|
||||
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
#[cfg(feature = "fast")]
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use litellm_host_python::{enter_native, release_gil};
|
||||
#[cfg(feature = "fast")]
|
||||
use litellm_token_counter::fast::{FastCounter, FastTokenizer};
|
||||
use litellm_token_counter::{Error, TextCodec};
|
||||
use pyo3::{exceptions::PyUnicodeEncodeError, prelude::*, types::PyString};
|
||||
|
||||
#[cfg(any(feature = "tiktoken", feature = "huggingface"))]
|
||||
use pyo3::exceptions::PyValueError;
|
||||
#[cfg(feature = "huggingface")]
|
||||
use pyo3::{exceptions::PyIOError, types::PyDict};
|
||||
#[cfg(feature = "tiktoken")]
|
||||
use pyo3::{
|
||||
exceptions::{PyKeyError, PyRuntimeError},
|
||||
types::PyBytes,
|
||||
};
|
||||
|
||||
#[cfg(not(all(feature = "tiktoken", feature = "huggingface")))]
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
use crate::routes::token_counter::token_count_error_to_pyerr;
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
use litellm_token_counter::huggingface::{
|
||||
EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingStrategy,
|
||||
TruncationDirection, encoding_from_json, encoding_to_json,
|
||||
};
|
||||
#[cfg(feature = "tiktoken")]
|
||||
use litellm_token_counter::tiktoken::{TiktokenTokenizer, Vocabulary};
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
pub(crate) fn load_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<TiktokenTokenizer> {
|
||||
enter_native()?;
|
||||
let resource: std::path::PathBuf =
|
||||
PyModule::import(py, "litellm.litellm_core_utils.tokenizers")?
|
||||
.getattr("__file__")?
|
||||
.extract()?;
|
||||
release_gil(py, || {
|
||||
TiktokenTokenizer::from_cached_ranks(encoding, |file| {
|
||||
std::fs::read_to_string(resource.with_file_name(file))
|
||||
})
|
||||
})
|
||||
.map_err(|error| token_count_error_to_pyerr(error.into()))
|
||||
}
|
||||
|
||||
pub(crate) enum Codec {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Tiktoken(TiktokenTokenizer),
|
||||
#[cfg(feature = "huggingface")]
|
||||
HuggingFace(HuggingFaceTokenizer),
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub(crate) fn codec(&self) -> &dyn TextCodec {
|
||||
match *self {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Self::Tiktoken(ref tokenizer) => tokenizer,
|
||||
#[cfg(feature = "huggingface")]
|
||||
Self::HuggingFace(ref tokenizer) => tokenizer,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "fast")]
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
match *self {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Self::Tiktoken(ref tokenizer) => tokenizer.fast_counter(),
|
||||
#[cfg(feature = "huggingface")]
|
||||
Self::HuggingFace(ref tokenizer) => tokenizer.fast_counter(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The loaded model is shared: `TokenCounter::from_tokenizer` counts with the same parse,
|
||||
/// and the opt-in count-only counter is derived from it once, on first use.
|
||||
#[pyclass(frozen, module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct Tokenizer {
|
||||
inner: Arc<Codec>,
|
||||
#[cfg(feature = "fast")]
|
||||
fast: OnceLock<Option<Arc<FastTokenizer>>>,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl Tokenizer {
|
||||
#[staticmethod]
|
||||
fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
{
|
||||
let tokenizer = load_tiktoken(py, encoding)?;
|
||||
Ok(Self::new(Codec::Tiktoken(tokenizer)))
|
||||
}
|
||||
#[cfg(not(feature = "tiktoken"))]
|
||||
{
|
||||
let _ = (py, encoding);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the tiktoken feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
fn from_json(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
|
||||
#[cfg(feature = "huggingface")]
|
||||
{
|
||||
enter_native()?;
|
||||
let tokenizer = release_gil(py, || HuggingFaceTokenizer::from_json(tokenizer_json))
|
||||
.map_err(|error| token_count_error_to_pyerr(error.into()))?;
|
||||
Ok(Self::new(Codec::HuggingFace(tokenizer)))
|
||||
}
|
||||
#[cfg(not(feature = "huggingface"))]
|
||||
{
|
||||
let _ = (py, tokenizer_json);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (identifier, revision = "main", token = None))]
|
||||
fn from_pretrained(
|
||||
py: Python<'_>,
|
||||
identifier: &str,
|
||||
revision: &str,
|
||||
token: Option<&str>,
|
||||
) -> PyResult<Self> {
|
||||
#[cfg(feature = "huggingface")]
|
||||
{
|
||||
enter_native()?;
|
||||
let kwargs = PyDict::new(py);
|
||||
kwargs.set_item("repo_id", identifier)?;
|
||||
kwargs.set_item("filename", "tokenizer.json")?;
|
||||
kwargs.set_item("revision", revision)?;
|
||||
kwargs.set_item("token", token)?;
|
||||
let path: String = PyModule::import(py, "huggingface_hub")?
|
||||
.getattr("hf_hub_download")?
|
||||
.call((), Some(&kwargs))?
|
||||
.extract()?;
|
||||
let json =
|
||||
release_gil(py, || std::fs::read_to_string(path)).map_err(PyIOError::new_err)?;
|
||||
Self::from_json(py, &json)
|
||||
}
|
||||
#[cfg(not(feature = "huggingface"))]
|
||||
{
|
||||
let _ = (py, identifier, revision, token);
|
||||
Err(RustBridgeDeclined::new_err(
|
||||
"tokenizer backend requires the huggingface feature",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn encode(&self, py: Python<'_>, text: &Bound<'_, PyString>) -> PyResult<Vec<u32>> {
|
||||
enter_native()?;
|
||||
let text = self.text(text)?;
|
||||
release_gil(py, || self.inner.codec().encode(&text)).map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (ids, skip_special_tokens = true))]
|
||||
fn decode(&self, py: Python<'_>, ids: Vec<u32>, skip_special_tokens: bool) -> PyResult<String> {
|
||||
enter_native()?;
|
||||
release_gil(py, || self.inner.codec().decode(&ids, skip_special_tokens))
|
||||
.map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (text, fast = false))]
|
||||
fn count(&self, py: Python<'_>, text: &Bound<'_, PyString>, fast: bool) -> PyResult<usize> {
|
||||
enter_native()?;
|
||||
let text = self.text(text)?;
|
||||
let counter = self.counter(py, fast);
|
||||
release_gil(py, || {
|
||||
litellm_token_counter::Tokenizer::count_tokens(&counter, &text)
|
||||
})
|
||||
.map_err(token_count_error_to_pyerr)
|
||||
}
|
||||
|
||||
#[getter]
|
||||
fn name(&self) -> &str {
|
||||
self.inner.codec().name()
|
||||
}
|
||||
|
||||
// ---- tiktoken: the `tiktoken.Encoding` surface ------------------------------------------
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_special(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
text: &Bound<'_, PyString>,
|
||||
allowed: Vec<String>,
|
||||
) -> PyResult<Vec<u32>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let text = self.text(text)?;
|
||||
release_gil(py, || tokenizer.encode_special(&text, &allowed))
|
||||
.map_err(PyRuntimeError::new_err)
|
||||
}
|
||||
|
||||
/// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_with_unstable(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
text: &Bound<'_, PyString>,
|
||||
allowed: Vec<String>,
|
||||
) -> PyResult<(Vec<u32>, Vec<Vec<u32>>)> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let text = self.text(text)?;
|
||||
Ok(release_gil(py, || {
|
||||
tokenizer.encode_with_unstable(&text, &allowed)
|
||||
}))
|
||||
}
|
||||
|
||||
/// The special tokens by text: tiktoken's `_special_tokens`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn special_tokens(&self) -> PyResult<HashMap<String, u32>> {
|
||||
Ok(self
|
||||
.vocabulary()?
|
||||
.special_tokens()
|
||||
.map(|(token, rank)| (token.to_owned(), rank))
|
||||
.collect())
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn max_token_value(&self) -> PyResult<u32> {
|
||||
Ok(self.vocabulary()?.max_token_value())
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn is_special_token(&self, token: u32) -> PyResult<bool> {
|
||||
Ok(self.vocabulary()?.is_special_token(token))
|
||||
}
|
||||
|
||||
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn token_byte_values<'py>(&self, py: Python<'py>) -> PyResult<Vec<Bound<'py, PyBytes>>> {
|
||||
let vocabulary = self.vocabulary()?;
|
||||
let values = release_gil(py, || vocabulary.token_byte_values());
|
||||
Ok(values.iter().map(|value| PyBytes::new(py, value)).collect())
|
||||
}
|
||||
|
||||
/// The token of one whole piece; `KeyError` when it is not in the vocabulary.
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn encode_single_token(&self, py: Python<'_>, piece: Vec<u8>) -> PyResult<u32> {
|
||||
self.vocabulary()?
|
||||
.encode_single_token(&piece)
|
||||
.ok_or_else(|| PyKeyError::new_err(PyBytes::new(py, &piece).unbind()))
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec<u32>) -> PyResult<Bound<'py, PyBytes>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.tiktoken()?;
|
||||
let bytes =
|
||||
release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?;
|
||||
Ok(PyBytes::new(py, &bytes))
|
||||
}
|
||||
|
||||
// ---- Hugging Face: the `tokenizers.Tokenizer` surface -----------------------------------
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (sequence, pair = None, is_pretokenized = false, add_special_tokens = true, fast = false))]
|
||||
fn encode_huggingface(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
sequence: Sequence,
|
||||
pair: Option<Sequence>,
|
||||
is_pretokenized: bool,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> PyResult<HuggingFaceEncoding> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
let sequence = sequence.input(is_pretokenized)?;
|
||||
let input = match pair {
|
||||
Some(pair) => EncodeInput::Dual(sequence, pair.input(is_pretokenized)?),
|
||||
None => EncodeInput::Single(sequence),
|
||||
};
|
||||
release_gil(py, || {
|
||||
tokenizer.encode_result(input, add_special_tokens, fast)
|
||||
})
|
||||
.map(|inner| HuggingFaceEncoding { inner })
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (inputs, is_pretokenized = false, add_special_tokens = true, fast = false))]
|
||||
fn encode_batch_huggingface(
|
||||
&self,
|
||||
py: Python<'_>,
|
||||
inputs: Vec<(Sequence, Option<Sequence>)>,
|
||||
is_pretokenized: bool,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> PyResult<Vec<HuggingFaceEncoding>> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
let inputs = inputs
|
||||
.into_iter()
|
||||
.map(|(sequence, pair)| {
|
||||
let sequence = sequence.input(is_pretokenized)?;
|
||||
match pair {
|
||||
Some(pair) => Ok(EncodeInput::Dual(sequence, pair.input(is_pretokenized)?)),
|
||||
None => Ok(EncodeInput::Single(sequence)),
|
||||
}
|
||||
})
|
||||
.collect::<PyResult<Vec<_>>>()?;
|
||||
release_gil(py, || {
|
||||
tokenizer.encode_batch_result(inputs, add_special_tokens, fast)
|
||||
})
|
||||
.map(|encodings| {
|
||||
encodings
|
||||
.into_iter()
|
||||
.map(|inner| HuggingFaceEncoding { inner })
|
||||
.collect()
|
||||
})
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (pretty = false))]
|
||||
fn to_json(&self, py: Python<'_>, pretty: bool) -> PyResult<String> {
|
||||
enter_native()?;
|
||||
let tokenizer = self.huggingface()?;
|
||||
release_gil(py, || tokenizer.to_json(pretty))
|
||||
.map_err(|error| token_count_error_to_pyerr(Error::from(error)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn token_to_id(&self, token: &str) -> PyResult<Option<u32>> {
|
||||
Ok(self.huggingface()?.token_to_id(token))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn id_to_token(&self, id: u32) -> PyResult<Option<String>> {
|
||||
Ok(self.huggingface()?.id_to_token(id))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (with_added_tokens = true))]
|
||||
fn get_vocab(&self, py: Python<'_>, with_added_tokens: bool) -> PyResult<HashMap<String, u32>> {
|
||||
let tokenizer = self.huggingface()?;
|
||||
Ok(release_gil(py, || tokenizer.vocab(with_added_tokens)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyo3(signature = (with_added_tokens = true))]
|
||||
fn get_vocab_size(&self, with_added_tokens: bool) -> PyResult<usize> {
|
||||
Ok(self.huggingface()?.vocab_size(with_added_tokens))
|
||||
}
|
||||
|
||||
/// The added tokens by id as `(id, (content, single_word, lstrip, rstrip, normalized,
|
||||
/// special))`, for Python to rebuild as `tokenizers.AddedToken`.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn added_tokens_decoder(&self) -> PyResult<Vec<(u32, AddedTokenFields)>> {
|
||||
Ok(self
|
||||
.huggingface()?
|
||||
.added_tokens_decoder()
|
||||
.into_iter()
|
||||
.map(|(id, token)| {
|
||||
(
|
||||
id,
|
||||
(
|
||||
token.content,
|
||||
token.single_word,
|
||||
token.lstrip,
|
||||
token.rstrip,
|
||||
token.normalized,
|
||||
token.special,
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// The padding parameters as `tokenizers.Tokenizer.padding` reports them.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn padding<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let Some(params) = self.huggingface()?.padding() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let padding = PyDict::new(py);
|
||||
padding.set_item(
|
||||
"length",
|
||||
match params.strategy {
|
||||
PaddingStrategy::BatchLongest => None,
|
||||
PaddingStrategy::Fixed(length) => Some(length),
|
||||
},
|
||||
)?;
|
||||
padding.set_item("pad_to_multiple_of", params.pad_to_multiple_of)?;
|
||||
padding.set_item("pad_id", params.pad_id)?;
|
||||
padding.set_item("pad_type_id", params.pad_type_id)?;
|
||||
padding.set_item("pad_token", ¶ms.pad_token)?;
|
||||
padding.set_item("direction", params.direction.as_ref())?;
|
||||
Ok(Some(padding))
|
||||
}
|
||||
|
||||
/// The truncation parameters as `tokenizers.Tokenizer.truncation` reports them.
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn truncation<'py>(&self, py: Python<'py>) -> PyResult<Option<Bound<'py, PyDict>>> {
|
||||
let Some(params) = self.huggingface()?.truncation() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let truncation = PyDict::new(py);
|
||||
truncation.set_item("max_length", params.max_length)?;
|
||||
truncation.set_item("stride", params.stride)?;
|
||||
truncation.set_item("strategy", params.strategy.as_ref())?;
|
||||
truncation.set_item("direction", params.direction.as_ref())?;
|
||||
Ok(Some(truncation))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn num_special_tokens_to_add(&self, is_pair: bool) -> PyResult<usize> {
|
||||
Ok(self.huggingface()?.num_special_tokens_to_add(is_pair))
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn encode_special_tokens(&self) -> PyResult<bool> {
|
||||
Ok(self.huggingface()?.encode_special_tokens())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
type AddedTokenFields = (String, bool, bool, bool, bool, bool);
|
||||
|
||||
impl Tokenizer {
|
||||
fn new(inner: Codec) -> Self {
|
||||
Self {
|
||||
inner: Arc::new(inner),
|
||||
#[cfg(feature = "fast")]
|
||||
fast: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn counter(&self, py: Python<'_>, fast: bool) -> SharedCounter {
|
||||
#[cfg(feature = "fast")]
|
||||
if fast {
|
||||
let counter = self.fast.get().unwrap_or_else(|| {
|
||||
release_gil(py, || {
|
||||
self.fast
|
||||
.get_or_init(|| self.inner.fast_counter().map(Arc::new))
|
||||
})
|
||||
});
|
||||
if let Some(counter) = counter {
|
||||
return SharedCounter::Fast(Arc::clone(counter));
|
||||
}
|
||||
}
|
||||
#[cfg(not(feature = "fast"))]
|
||||
let _ = (py, fast);
|
||||
SharedCounter::Codec(Arc::clone(&self.inner))
|
||||
}
|
||||
|
||||
/// A Python `str` as UTF-8. tiktoken replaces lone surrogates the way its Python `encode`
|
||||
/// does; `tokenizers` rejects them, so that backend keeps the encode error.
|
||||
fn text<'a>(&self, text: &'a Bound<'_, PyString>) -> PyResult<Cow<'a, str>> {
|
||||
match text.to_cow() {
|
||||
Ok(text) => Ok(text),
|
||||
Err(error) => match *self.inner {
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Codec::Tiktoken(_) if error.is_instance_of::<PyUnicodeEncodeError>(text.py()) => {
|
||||
text.call_method1("encode", ("utf-16", "surrogatepass"))?
|
||||
.call_method1("decode", ("utf-16", "replace"))?
|
||||
.extract::<String>()
|
||||
.map(Cow::Owned)
|
||||
}
|
||||
_ => Err(error),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn tiktoken(&self) -> PyResult<&TiktokenTokenizer> {
|
||||
match *self.inner {
|
||||
Codec::Tiktoken(ref tokenizer) => Ok(tokenizer),
|
||||
#[cfg(feature = "huggingface")]
|
||||
Codec::HuggingFace(_) => Err(PyValueError::new_err("requires a tiktoken encoding")),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
fn vocabulary(&self) -> PyResult<&Vocabulary> {
|
||||
self.tiktoken()?.vocabulary().ok_or_else(|| {
|
||||
PyRuntimeError::new_err("this encoding was built without its vocabulary")
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn huggingface(&self) -> PyResult<&HuggingFaceTokenizer> {
|
||||
match *self.inner {
|
||||
Codec::HuggingFace(ref tokenizer) => Ok(tokenizer),
|
||||
#[cfg(feature = "tiktoken")]
|
||||
Codec::Tiktoken(_) => Err(PyValueError::new_err("requires a Hugging Face tokenizer")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) enum SharedCounter {
|
||||
Codec(Arc<Codec>),
|
||||
#[cfg(feature = "fast")]
|
||||
Fast(Arc<FastTokenizer>),
|
||||
}
|
||||
|
||||
impl litellm_token_counter::Tokenizer for SharedCounter {
|
||||
fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
match self {
|
||||
Self::Codec(codec) => codec.codec().count_tokens(text),
|
||||
#[cfg(feature = "fast")]
|
||||
Self::Fast(counter) => counter.count_tokens(text).map_err(Error::from),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[derive(FromPyObject)]
|
||||
pub(crate) enum Sequence {
|
||||
Text(String),
|
||||
Words(Vec<String>),
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
impl Sequence {
|
||||
fn input(self, is_pretokenized: bool) -> PyResult<InputSequence<'static>> {
|
||||
match (self, is_pretokenized) {
|
||||
(Self::Text(text), false) => Ok(text.into()),
|
||||
(Self::Words(words), true) => Ok(words.into()),
|
||||
_ => Err(pyo3::exceptions::PyTypeError::new_err(
|
||||
"input must match is_pretokenized",
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
fn direction<T>(value: &str, left: T, right: T, what: &str) -> PyResult<T> {
|
||||
match value {
|
||||
"left" => Ok(left),
|
||||
"right" => Ok(right),
|
||||
other => Err(PyValueError::new_err(format!(
|
||||
"invalid {what} direction {other:?}: expected 'left' or 'right'"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// `tokenizers.Encoding`, mutable like the original: `pad`, `truncate` and `set_sequence_id`
|
||||
/// change it in place.
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pyclass(module = "litellm.rust_bridge._native")]
|
||||
pub(crate) struct HuggingFaceEncoding {
|
||||
inner: Encoding,
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
#[pymethods]
|
||||
impl HuggingFaceEncoding {
|
||||
#[new]
|
||||
#[pyo3(signature = (json = None))]
|
||||
fn new(json: Option<&str>) -> PyResult<Self> {
|
||||
let inner = match json {
|
||||
Some(json) => encoding_from_json(json)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?,
|
||||
None => Encoding::default(),
|
||||
};
|
||||
Ok(Self { inner })
|
||||
}
|
||||
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (encodings, growing_offsets = true))]
|
||||
fn merge(encodings: Vec<PyRef<'_, Self>>, growing_offsets: bool) -> Self {
|
||||
Self {
|
||||
inner: Encoding::merge(
|
||||
encodings.iter().map(|encoding| encoding.inner.clone()),
|
||||
growing_offsets,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn __reduce__<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> {
|
||||
let json = encoding_to_json(&self.inner)
|
||||
.map_err(|error| PyValueError::new_err(error.to_string()))?;
|
||||
Ok((py.get_type::<Self>(), (json,)))
|
||||
}
|
||||
|
||||
fn __repr__(&self) -> String {
|
||||
format!(
|
||||
"Encoding(num_tokens={}, attributes=[ids, type_ids, tokens, offsets, \
|
||||
attention_mask, special_tokens_mask, overflowing])",
|
||||
self.inner.len()
|
||||
)
|
||||
}
|
||||
|
||||
fn __len__(&self) -> usize {
|
||||
self.inner.len()
|
||||
}
|
||||
#[getter]
|
||||
fn ids(&self) -> Vec<u32> {
|
||||
self.inner.get_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn tokens(&self) -> Vec<String> {
|
||||
self.inner.get_tokens().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn offsets(&self) -> Vec<(usize, usize)> {
|
||||
self.inner.get_offsets().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn type_ids(&self) -> Vec<u32> {
|
||||
self.inner.get_type_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn attention_mask(&self) -> Vec<u32> {
|
||||
self.inner.get_attention_mask().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn special_tokens_mask(&self) -> Vec<u32> {
|
||||
self.inner.get_special_tokens_mask().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn word_ids(&self) -> Vec<Option<u32>> {
|
||||
self.inner.get_word_ids().to_vec()
|
||||
}
|
||||
#[getter]
|
||||
fn sequence_ids(&self) -> Vec<Option<usize>> {
|
||||
self.inner.get_sequence_ids()
|
||||
}
|
||||
#[getter]
|
||||
fn overflowing(&self) -> Vec<Self> {
|
||||
self.inner
|
||||
.get_overflowing()
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|inner| Self { inner })
|
||||
.collect()
|
||||
}
|
||||
#[getter]
|
||||
fn n_sequences(&self) -> usize {
|
||||
self.inner.n_sequences()
|
||||
}
|
||||
|
||||
#[pyo3(signature = (word_index, sequence_index = 0))]
|
||||
fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner.word_to_tokens(word_index, sequence_index)
|
||||
}
|
||||
#[pyo3(signature = (word_index, sequence_index = 0))]
|
||||
fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner.word_to_chars(word_index, sequence_index)
|
||||
}
|
||||
fn token_to_sequence(&self, token_index: usize) -> Option<usize> {
|
||||
self.inner.token_to_sequence(token_index)
|
||||
}
|
||||
fn token_to_chars(&self, token_index: usize) -> Option<(usize, usize)> {
|
||||
self.inner
|
||||
.token_to_chars(token_index)
|
||||
.map(|(_, offsets)| offsets)
|
||||
}
|
||||
fn token_to_word(&self, token_index: usize) -> Option<u32> {
|
||||
self.inner.token_to_word(token_index).map(|(_, word)| word)
|
||||
}
|
||||
#[pyo3(signature = (char_pos, sequence_index = 0))]
|
||||
fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option<usize> {
|
||||
self.inner.char_to_token(char_pos, sequence_index)
|
||||
}
|
||||
#[pyo3(signature = (char_pos, sequence_index = 0))]
|
||||
fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option<u32> {
|
||||
self.inner.char_to_word(char_pos, sequence_index)
|
||||
}
|
||||
|
||||
fn set_sequence_id(&mut self, sequence_id: usize) {
|
||||
self.inner.set_sequence_id(sequence_id);
|
||||
}
|
||||
|
||||
#[pyo3(signature = (length, direction = "right", pad_id = 0, pad_type_id = 0, pad_token = "[PAD]"))]
|
||||
fn pad(
|
||||
&mut self,
|
||||
length: usize,
|
||||
direction: &str,
|
||||
pad_id: u32,
|
||||
pad_type_id: u32,
|
||||
pad_token: &str,
|
||||
) -> PyResult<()> {
|
||||
let direction = self::direction(
|
||||
direction,
|
||||
PaddingDirection::Left,
|
||||
PaddingDirection::Right,
|
||||
"padding",
|
||||
)?;
|
||||
self.inner
|
||||
.pad(length, pad_id, pad_type_id, pad_token, direction);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[pyo3(signature = (max_length, stride = 0, direction = "right"))]
|
||||
fn truncate(&mut self, max_length: usize, stride: usize, direction: &str) -> PyResult<()> {
|
||||
let direction = self::direction(
|
||||
direction,
|
||||
TruncationDirection::Left,
|
||||
TruncationDirection::Right,
|
||||
"truncation",
|
||||
)?;
|
||||
self.inner.truncate(max_length, stride, direction);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,10 @@ pub enum Error {
|
|||
Auth(#[from] #[redact] litellm_auth_aws::Error),
|
||||
#[error("AWS region is not configured")]
|
||||
MissingRegion,
|
||||
#[error("AWS Secrets Manager received a non-AWS operation context")]
|
||||
InvalidOperationContext,
|
||||
#[error("AWS Secrets Manager was constructed without context-aware configuration")]
|
||||
OperationContextUnavailable,
|
||||
#[error("KMS response has no plaintext")]
|
||||
MissingPlaintext,
|
||||
#[error("AWS request timed out")]
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ use litellm_auth_aws::constants::{
|
|||
};
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secrets_types::{
|
||||
BaseSecretManager, KeyManagementSettings, Secret, SecretValue, async_rotate_secret,
|
||||
AwsOperationContext, BaseSecretManager, KeyManagementSettings, Secret, SecretOperationContext,
|
||||
SecretValue, SecretWriteContext, async_rotate_secret,
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -25,9 +26,17 @@ use crate::{Error, auth};
|
|||
#[derive(Clone)]
|
||||
pub struct AwsSecretsManagerV2 {
|
||||
client: Client,
|
||||
context_client_factory: Option<Box<ContextClientFactory>>,
|
||||
write_settings: AwsSecretWriteSettings,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ContextClientFactory {
|
||||
settings: KeyManagementSettings,
|
||||
environment: Arc<dyn Lookup + Send + Sync>,
|
||||
endpoint_url: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct AwsSecretWriteSettings {
|
||||
pub kms_key_id: Option<String>,
|
||||
|
|
@ -55,6 +64,19 @@ impl AwsSecretsManagerV2 {
|
|||
pub fn new(client: Client, write_settings: AwsSecretWriteSettings) -> Self {
|
||||
Self {
|
||||
client,
|
||||
context_client_factory: None,
|
||||
write_settings,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_context_client_factory(
|
||||
client: Client,
|
||||
write_settings: AwsSecretWriteSettings,
|
||||
context_client_factory: ContextClientFactory,
|
||||
) -> Self {
|
||||
Self {
|
||||
client,
|
||||
context_client_factory: Some(Box::new(context_client_factory)),
|
||||
write_settings,
|
||||
}
|
||||
}
|
||||
|
|
@ -67,19 +89,18 @@ impl AwsSecretsManagerV2 {
|
|||
if use_aws_secret_manager != Some(true) {
|
||||
return Ok(None);
|
||||
}
|
||||
let builder = aws_sdk_secretsmanager::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(auth::region(&settings, environment.as_ref())?))
|
||||
.credentials_provider(auth::Credentials::new(&settings, environment.clone()));
|
||||
let config = match environment.get(AWS_BEDROCK_RUNTIME_ENDPOINT) {
|
||||
Some(url) => builder
|
||||
.endpoint_url(url.replace("bedrock-runtime", "secretsmanager"))
|
||||
.build(),
|
||||
None => builder.build(),
|
||||
let context_client_factory = ContextClientFactory {
|
||||
settings: settings.clone(),
|
||||
environment: environment.clone(),
|
||||
endpoint_url: environment
|
||||
.get(AWS_BEDROCK_RUNTIME_ENDPOINT)
|
||||
.map(|url| url.replace("bedrock-runtime", "secretsmanager")),
|
||||
};
|
||||
Ok(Some(Self::new(
|
||||
Client::from_conf(config),
|
||||
let client = context_client_factory.client(&AwsOperationContext::default())?;
|
||||
Ok(Some(Self::with_context_client_factory(
|
||||
client,
|
||||
(&settings).into(),
|
||||
context_client_factory,
|
||||
)))
|
||||
}
|
||||
|
||||
|
|
@ -118,7 +139,14 @@ impl AwsSecretsManagerV2 {
|
|||
}
|
||||
|
||||
pub async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
|
||||
match self.client.get_secret_value().secret_id(name).send().await {
|
||||
Self::async_read_secret_with_client(&self.client, name).await
|
||||
}
|
||||
|
||||
async fn async_read_secret_with_client(
|
||||
client: &Client,
|
||||
name: &str,
|
||||
) -> Result<Option<SecretValue>, Error> {
|
||||
match client.get_secret_value().secret_id(name).send().await {
|
||||
Ok(response) => response
|
||||
.secret_string
|
||||
.map(SecretValue::new)
|
||||
|
|
@ -149,8 +177,19 @@ impl AwsSecretsManagerV2 {
|
|||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
) -> Result<CreateSecretOutput, Error> {
|
||||
let response = self
|
||||
.client
|
||||
self.async_write_secret_with_client_and_tags(&self.client, name, value, description, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn async_write_secret_with_client_and_tags(
|
||||
&self,
|
||||
client: &Client,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
tags: Option<&BTreeMap<String, String>>,
|
||||
) -> Result<CreateSecretOutput, Error> {
|
||||
let response = client
|
||||
.create_secret()
|
||||
.name(name)
|
||||
.secret_string(value.expose())
|
||||
|
|
@ -161,7 +200,7 @@ impl AwsSecretsManagerV2 {
|
|||
.clone()
|
||||
.filter(|v| !v.is_empty()),
|
||||
)
|
||||
.set_tags(self.write_settings.tags.as_ref().map(|tags| {
|
||||
.set_tags(tags.or(self.write_settings.tags.as_ref()).map(|tags| {
|
||||
tags.iter()
|
||||
.map(|(key, value)| Tag::builder().key(key).value(value).build())
|
||||
.collect()
|
||||
|
|
@ -171,7 +210,10 @@ impl AwsSecretsManagerV2 {
|
|||
.map_err(|error| Error::Create(Box::new(error)))?;
|
||||
if let Some(regions) = &self.write_settings.replica_regions
|
||||
&& !regions.is_empty()
|
||||
&& self.async_replicate_secret(name, regions).await.is_err()
|
||||
&& self
|
||||
.async_replicate_secret_with_client(client, name, regions)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!("secret created but replication failed");
|
||||
}
|
||||
|
|
@ -182,11 +224,21 @@ impl AwsSecretsManagerV2 {
|
|||
&self,
|
||||
name: &str,
|
||||
regions: &[String],
|
||||
) -> Result<Option<ReplicateSecretToRegionsOutput>, Error> {
|
||||
self.async_replicate_secret_with_client(&self.client, name, regions)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn async_replicate_secret_with_client(
|
||||
&self,
|
||||
client: &Client,
|
||||
name: &str,
|
||||
regions: &[String],
|
||||
) -> Result<Option<ReplicateSecretToRegionsOutput>, Error> {
|
||||
if regions.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
self.client
|
||||
client
|
||||
.replicate_secret_to_regions()
|
||||
.secret_id(name)
|
||||
.set_add_replica_regions(Some(
|
||||
|
|
@ -206,7 +258,17 @@ impl AwsSecretsManagerV2 {
|
|||
name: &str,
|
||||
value: &SecretValue,
|
||||
) -> Result<PutSecretValueOutput, Error> {
|
||||
self.client
|
||||
self.async_put_secret_value_with_client(&self.client, name, value)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn async_put_secret_value_with_client(
|
||||
&self,
|
||||
client: &Client,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
) -> Result<PutSecretValueOutput, Error> {
|
||||
client
|
||||
.put_secret_value()
|
||||
.secret_id(name)
|
||||
.secret_string(value.expose())
|
||||
|
|
@ -218,12 +280,22 @@ impl AwsSecretsManagerV2 {
|
|||
pub async fn async_delete_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
recovery_window_in_days: i64,
|
||||
recovery_window_in_days: Option<u32>,
|
||||
) -> Result<DeleteSecretOutput, Error> {
|
||||
self.client
|
||||
self.async_delete_secret_with_client(&self.client, name, recovery_window_in_days)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn async_delete_secret_with_client(
|
||||
&self,
|
||||
client: &Client,
|
||||
name: &str,
|
||||
recovery_window_in_days: Option<u32>,
|
||||
) -> Result<DeleteSecretOutput, Error> {
|
||||
client
|
||||
.delete_secret()
|
||||
.secret_id(name)
|
||||
.recovery_window_in_days(recovery_window_in_days)
|
||||
.set_recovery_window_in_days(recovery_window_in_days.map(i64::from))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| Error::Delete(Box::new(error)))
|
||||
|
|
@ -234,17 +306,105 @@ impl AwsSecretsManagerV2 {
|
|||
current_name: &str,
|
||||
new_name: &str,
|
||||
value: &SecretValue,
|
||||
) -> Result<RotationResponse, Error> {
|
||||
self.async_rotate_secret_with_context(
|
||||
current_name,
|
||||
new_name,
|
||||
value,
|
||||
&SecretOperationContext::default(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn async_rotate_secret_with_context(
|
||||
&self,
|
||||
current_name: &str,
|
||||
new_name: &str,
|
||||
value: &SecretValue,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<RotationResponse, Error> {
|
||||
if current_name == new_name {
|
||||
let client = self.client_for_context(context)?;
|
||||
return self
|
||||
.async_put_secret_value(current_name, value)
|
||||
.async_put_secret_value_with_client(&client, current_name, value)
|
||||
.await
|
||||
.map(RotationResponse::Updated);
|
||||
}
|
||||
async_rotate_secret(self, current_name, new_name, value)
|
||||
async_rotate_secret(self, current_name, new_name, value, context)
|
||||
.await
|
||||
.map(RotationResponse::Created)
|
||||
}
|
||||
|
||||
fn client_for_context(&self, context: &SecretOperationContext) -> Result<Client, Error> {
|
||||
match context {
|
||||
SecretOperationContext::Default => Ok(self.client.clone()),
|
||||
SecretOperationContext::Aws(context) if context == &AwsOperationContext::default() => {
|
||||
Ok(self.client.clone())
|
||||
}
|
||||
SecretOperationContext::Aws(context) => self
|
||||
.context_client_factory
|
||||
.as_ref()
|
||||
.ok_or(Error::OperationContextUnavailable)?
|
||||
.client(context),
|
||||
_ => Err(Error::InvalidOperationContext),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ContextClientFactory {
|
||||
fn client(&self, context: &AwsOperationContext) -> Result<Client, Error> {
|
||||
let settings = KeyManagementSettings {
|
||||
aws_region_name: context
|
||||
.region_name
|
||||
.clone()
|
||||
.or_else(|| self.settings.aws_region_name.clone()),
|
||||
aws_role_name: context
|
||||
.role_name
|
||||
.clone()
|
||||
.or_else(|| self.settings.aws_role_name.clone()),
|
||||
aws_session_name: context
|
||||
.session_name
|
||||
.clone()
|
||||
.or_else(|| self.settings.aws_session_name.clone()),
|
||||
aws_external_id: context
|
||||
.external_id
|
||||
.clone()
|
||||
.or_else(|| self.settings.aws_external_id.clone()),
|
||||
aws_profile_name: context
|
||||
.profile_name
|
||||
.clone()
|
||||
.or_else(|| self.settings.aws_profile_name.clone()),
|
||||
aws_web_identity_token: context
|
||||
.web_identity_token
|
||||
.clone()
|
||||
.or_else(|| self.settings.aws_web_identity_token.clone()),
|
||||
aws_sts_endpoint: context
|
||||
.sts_endpoint
|
||||
.clone()
|
||||
.or_else(|| self.settings.aws_sts_endpoint.clone()),
|
||||
..self.settings.clone()
|
||||
};
|
||||
let builder = aws_sdk_secretsmanager::Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(auth::region(
|
||||
&settings,
|
||||
self.environment.as_ref(),
|
||||
)?))
|
||||
.credentials_provider(auth::Credentials::new(&settings, self.environment.clone()));
|
||||
let builder = match context.timeout {
|
||||
Some(timeout) => builder.timeout_config(
|
||||
aws_sdk_secretsmanager::config::timeout::TimeoutConfig::builder()
|
||||
.operation_timeout(timeout)
|
||||
.build(),
|
||||
),
|
||||
None => builder,
|
||||
};
|
||||
let config = match &self.endpoint_url {
|
||||
Some(endpoint_url) => builder.endpoint_url(endpoint_url.clone()).build(),
|
||||
None => builder.build(),
|
||||
};
|
||||
Ok(Client::from_conf(config))
|
||||
}
|
||||
}
|
||||
|
||||
impl BaseSecretManager for AwsSecretsManagerV2 {
|
||||
|
|
@ -252,25 +412,40 @@ impl BaseSecretManager for AwsSecretsManagerV2 {
|
|||
type WriteResponse = CreateSecretOutput;
|
||||
type DeleteResponse = DeleteSecretOutput;
|
||||
|
||||
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
|
||||
self.async_read_secret(name).await
|
||||
async fn async_read_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<SecretValue>, Error> {
|
||||
let client = self.client_for_context(context)?;
|
||||
Self::async_read_secret_with_client(&client, name).await
|
||||
}
|
||||
|
||||
async fn async_write_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
context: &SecretWriteContext,
|
||||
) -> Result<CreateSecretOutput, Error> {
|
||||
self.async_write_secret(name, value, description).await
|
||||
let client = self.client_for_context(&context.operation)?;
|
||||
self.async_write_secret_with_client_and_tags(
|
||||
&client,
|
||||
name,
|
||||
value,
|
||||
context.description.as_deref(),
|
||||
(!context.tags.is_empty()).then_some(&context.tags),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn async_delete_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
recovery_window_in_days: i64,
|
||||
recovery_window_in_days: Option<u32>,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<DeleteSecretOutput, Error> {
|
||||
self.async_delete_secret(name, recovery_window_in_days)
|
||||
let client = self.client_for_context(context)?;
|
||||
self.async_delete_secret_with_client(&client, name, recovery_window_in_days)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@ use aws_sdk_kms::{
|
|||
config::{BehaviorVersion, Credentials, Region, retry::RetryConfig},
|
||||
};
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use litellm_secrets_aws::AwsKms;
|
||||
use litellm_secrets_aws::{AwsKms, Error, load_aws_kms};
|
||||
use litellm_secrets_types::KeyManagementSettings;
|
||||
use rstest::rstest;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{body_json, header},
|
||||
};
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -40,20 +43,56 @@ async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_kms_loader_does_not_require_environment_configuration() {
|
||||
use litellm_secrets_aws::load_aws_kms;
|
||||
use litellm_secrets_types::KeyManagementSettings;
|
||||
#[rstest]
|
||||
#[case::unset(None)]
|
||||
#[case::disabled(Some(false))]
|
||||
fn disabled_kms_loader_does_not_require_environment_configuration(#[case] enabled: Option<bool>) {
|
||||
use std::sync::Arc;
|
||||
for enabled in [None, Some(false)] {
|
||||
assert!(
|
||||
load_aws_kms(
|
||||
enabled,
|
||||
&KeyManagementSettings::default(),
|
||||
Arc::new(|_: &str| None)
|
||||
)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
load_aws_kms(
|
||||
enabled,
|
||||
&KeyManagementSettings::default(),
|
||||
Arc::new(|_: &str| None)
|
||||
)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::settings(Some("configured-region"), None)]
|
||||
#[case::environment(None, Some("environment-region"))]
|
||||
fn enabled_kms_loader_accepts_either_region_source(
|
||||
#[case] configured_region: Option<&'static str>,
|
||||
#[case] environment_region: Option<&'static str>,
|
||||
) {
|
||||
use std::sync::Arc;
|
||||
let settings = KeyManagementSettings {
|
||||
aws_region_name: configured_region.map(str::to_owned),
|
||||
..KeyManagementSettings::default()
|
||||
};
|
||||
let environment = Arc::new(move |name: &str| {
|
||||
(name == "AWS_REGION_NAME")
|
||||
.then(|| environment_region.map(str::to_owned))
|
||||
.flatten()
|
||||
});
|
||||
|
||||
assert!(
|
||||
load_aws_kms(Some(true), &settings, environment)
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn enabled_kms_loader_rejects_missing_region() {
|
||||
use std::sync::Arc;
|
||||
assert!(matches!(
|
||||
load_aws_kms(
|
||||
Some(true),
|
||||
&KeyManagementSettings::default(),
|
||||
Arc::new(|_: &str| None),
|
||||
),
|
||||
Err(Error::MissingRegion)
|
||||
));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,21 @@
|
|||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use aws_sdk_secretsmanager::{
|
||||
Client,
|
||||
config::{BehaviorVersion, Credentials, Region, retry::RetryConfig},
|
||||
};
|
||||
use litellm_secrets_aws::{AwsSecretsManagerV2, Error, RotationResponse};
|
||||
use litellm_secrets_types::{KeyManagementSettings, SecretValue};
|
||||
use litellm_secrets_types::{
|
||||
AwsOperationContext, BaseSecretManager, KeyManagementSettings, SecretOperationContext,
|
||||
SecretValue, SecretWriteContext,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
|
|
@ -25,12 +35,39 @@ fn manager(server: &MockServer, settings: KeyManagementSettings) -> AwsSecretsMa
|
|||
AwsSecretsManagerV2::new(client, (&settings).into())
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
fn loaded_manager(server: &MockServer) -> AwsSecretsManagerV2 {
|
||||
let endpoint_url = server.uri();
|
||||
let environment: Arc<dyn litellm_core_utils::settings::Lookup + Send + Sync> =
|
||||
Arc::new(move |name: &str| match name {
|
||||
"AWS_BEDROCK_RUNTIME_ENDPOINT" => Some(endpoint_url.clone()),
|
||||
"AWS_ACCESS_KEY_ID" => Some("test".into()),
|
||||
"AWS_SECRET_ACCESS_KEY" => Some("test".into()),
|
||||
_ => None,
|
||||
});
|
||||
AwsSecretsManagerV2::load_aws_secret_manager(
|
||||
Some(true),
|
||||
KeyManagementSettings {
|
||||
aws_region_name: Some("us-east-1".into()),
|
||||
..Default::default()
|
||||
},
|
||||
environment,
|
||||
)
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn default_settings() -> KeyManagementSettings {
|
||||
KeyManagementSettings::default()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::string_value("KEY", Some("value"))]
|
||||
#[case::missing_value("missing", None)]
|
||||
#[case::non_string_value("BOOL", None)]
|
||||
#[tokio::test]
|
||||
async fn primary_lookup_preserves_read_semantics(
|
||||
default_settings: KeyManagementSettings,
|
||||
#[case] name: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
|
|
@ -45,7 +82,7 @@ async fn primary_lookup_preserves_read_semantics(
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, KeyManagementSettings::default());
|
||||
let manager = manager(&server, default_settings);
|
||||
assert_eq!(
|
||||
manager
|
||||
.read_secret_for_resolver(name, Some("primary"), &|_: &str| None)
|
||||
|
|
@ -57,16 +94,19 @@ async fn primary_lookup_preserves_read_semantics(
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::access_key("AWS_ACCESS_KEY_ID")]
|
||||
#[case::secret_access_key("AWS_SECRET_ACCESS_KEY")]
|
||||
#[case::region_name("AWS_REGION_NAME")]
|
||||
#[case::region("AWS_REGION")]
|
||||
#[case::bedrock_endpoint("AWS_BEDROCK_RUNTIME_ENDPOINT")]
|
||||
#[tokio::test]
|
||||
async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) {
|
||||
async fn bootstrap_keys_bypass_primary_lookup(
|
||||
default_settings: KeyManagementSettings,
|
||||
#[case] name: &str,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
let manager = manager(&server, KeyManagementSettings::default());
|
||||
let manager = manager(&server, default_settings);
|
||||
assert_eq!(
|
||||
manager
|
||||
.read_secret_for_resolver(name, Some("primary"), &|_: &str| Some("bootstrap".into()))
|
||||
|
|
@ -79,8 +119,11 @@ async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() {
|
||||
async fn failed_read_returns_none_but_invalid_primary_json_is_an_error(
|
||||
default_settings: KeyManagementSettings,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(body_partial_json(json!({"SecretId":"missing"})))
|
||||
.respond_with(
|
||||
|
|
@ -92,7 +135,11 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() {
|
|||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"SecretString":"not-json"})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, KeyManagementSettings::default());
|
||||
Mock::given(body_partial_json(json!({"SecretId":"no-string"})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name":"no-string"})))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, default_settings);
|
||||
assert!(
|
||||
manager
|
||||
.async_read_secret("missing")
|
||||
|
|
@ -106,10 +153,17 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() {
|
|||
.await,
|
||||
Err(Error::PrimarySecret)
|
||||
));
|
||||
assert!(matches!(
|
||||
manager.async_read_secret("no-string").await,
|
||||
Err(Error::MissingString)
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn same_name_rotation_uses_put_and_returns_its_response() {
|
||||
async fn same_name_rotation_uses_put_and_returns_its_response(
|
||||
default_settings: KeyManagementSettings,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.PutSecretValue"))
|
||||
.and(body_partial_json(
|
||||
|
|
@ -121,7 +175,7 @@ async fn same_name_rotation_uses_put_and_returns_its_response() {
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let response = manager(&server, KeyManagementSettings::default())
|
||||
let response = manager(&server, default_settings)
|
||||
.async_rotate_secret("key", "key", &SecretValue::new("replacement"))
|
||||
.await
|
||||
.unwrap();
|
||||
|
|
@ -132,8 +186,11 @@ async fn same_name_rotation_uses_put_and_returns_its_response() {
|
|||
assert_eq!(server.received_requests().await.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn renamed_rotation_reads_creates_verifies_then_deletes() {
|
||||
async fn renamed_rotation_reads_creates_verifies_then_deletes(
|
||||
default_settings: KeyManagementSettings,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
let step = AtomicUsize::new(0);
|
||||
Mock::given(wiremock::matchers::method("POST"))
|
||||
|
|
@ -176,7 +233,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() {
|
|||
.mount(&server)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
manager(&server, KeyManagementSettings::default())
|
||||
manager(&server, default_settings)
|
||||
.async_rotate_secret("old", "new", &SecretValue::new("replacement"))
|
||||
.await
|
||||
.unwrap(),
|
||||
|
|
@ -184,6 +241,7 @@ async fn renamed_rotation_reads_creates_verifies_then_deletes() {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn creation_passes_tags_and_kms_and_survives_replication_failure() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -230,6 +288,112 @@ async fn creation_passes_tags_and_kms_and_survives_replication_failure() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn trait_write_uses_typed_write_context(default_settings: KeyManagementSettings) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.CreateSecret"))
|
||||
.and(body_partial_json(json!({
|
||||
"Name": "key",
|
||||
"SecretString": "value",
|
||||
"Description": "created by caller",
|
||||
"Tags": [{"Key": "stage", "Value": "test"}],
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let context = SecretWriteContext {
|
||||
description: Some("created by caller".into()),
|
||||
tags: std::collections::BTreeMap::from([("stage".into(), "test".into())]),
|
||||
..Default::default()
|
||||
};
|
||||
let response = BaseSecretManager::async_write_secret(
|
||||
&manager(&server, default_settings),
|
||||
"key",
|
||||
&SecretValue::new("value"),
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.name(), Some("key"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn trait_delete_accepts_an_unspecified_recovery_window(
|
||||
default_settings: KeyManagementSettings,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.DeleteSecret"))
|
||||
.and(body_partial_json(json!({"SecretId": "key"})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name": "key"})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let response = BaseSecretManager::async_delete_secret(
|
||||
&manager(&server, default_settings),
|
||||
"key",
|
||||
None,
|
||||
&SecretOperationContext::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.name(), Some("key"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn trait_read_uses_the_aws_region_from_its_operation_context() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.respond_with(|request: &wiremock::Request| {
|
||||
let authorization = request
|
||||
.headers
|
||||
.get("authorization")
|
||||
.unwrap()
|
||||
.to_str()
|
||||
.unwrap();
|
||||
assert!(authorization.contains("/us-west-2/secretsmanager/aws4_request"));
|
||||
ResponseTemplate::new(200).set_body_json(json!({"SecretString": "value"}))
|
||||
})
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let context = SecretOperationContext::Aws(AwsOperationContext {
|
||||
region_name: Some("us-west-2".into()),
|
||||
..Default::default()
|
||||
});
|
||||
let value = BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(value.unwrap().expose(), "value");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn trait_read_applies_the_aws_operation_timeout() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_delay(Duration::from_secs(1))
|
||||
.set_body_json(json!({"SecretString": "late"})),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let context = SecretOperationContext::Aws(AwsOperationContext {
|
||||
timeout: Some(Duration::from_millis(30)),
|
||||
..Default::default()
|
||||
});
|
||||
assert!(matches!(
|
||||
BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context).await,
|
||||
Err(Error::Timeout)
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn credential_failures_are_not_swallowed_as_missing_secrets() {
|
||||
use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
|
||||
|
|
@ -260,9 +424,9 @@ async fn credential_failures_are_not_swallowed_as_missing_secrets() {
|
|||
assert!(server.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() {
|
||||
use std::time::Duration;
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(wiremock::matchers::method("POST"))
|
||||
.respond_with(
|
||||
|
|
@ -291,12 +455,16 @@ async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::denied(400, "AccessDeniedException")]
|
||||
#[case::throttled(400, "ThrottlingException")]
|
||||
#[case::unavailable(503, "ServiceUnavailableException")]
|
||||
#[tokio::test]
|
||||
async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str) {
|
||||
async fn service_failures_remain_errors(
|
||||
default_settings: KeyManagementSettings,
|
||||
#[case] status: u16,
|
||||
#[case] code: &str,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
|
||||
.respond_with(ResponseTemplate::new(status).set_body_json(json!({"__type":code})))
|
||||
|
|
@ -304,7 +472,7 @@ async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str)
|
|||
.mount(&server)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
manager(&server, KeyManagementSettings::default())
|
||||
manager(&server, default_settings)
|
||||
.async_read_secret("key")
|
||||
.await,
|
||||
Err(Error::Read(_))
|
||||
|
|
|
|||
|
|
@ -76,10 +76,7 @@ impl AzureKeyVault {
|
|||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
pub async fn get_secret_from_azure_key_vault(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<Option<Secret>, Error> {
|
||||
pub async fn get_secret(&self, name: &str) -> Result<Option<Secret>, Error> {
|
||||
let token = self
|
||||
.auth
|
||||
.get_azure_ad_token(&self.inputs, &|key| self.environment.get(key))
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ use std::sync::Arc;
|
|||
|
||||
use litellm_secrets_azure::{AzureKeyVault, Error};
|
||||
use litellm_secrets_types::{Secret, SecretValue};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde::Deserialize;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
|
|
@ -17,6 +18,7 @@ fn manager(server: &MockServer) -> AzureKeyVault {
|
|||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn reads_secret_with_bearer_token_and_api_version() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -32,7 +34,7 @@ async fn reads_secret_with_bearer_token_and_api_version() {
|
|||
.await;
|
||||
|
||||
let secret = manager(&server)
|
||||
.get_secret_from_azure_key_vault("OPENAI-API-KEY")
|
||||
.get_secret("OPENAI-API-KEY")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
|
@ -40,6 +42,7 @@ async fn reads_secret_with_bearer_token_and_api_version() {
|
|||
assert_eq!(secret, Secret::String(SecretValue::new("s3cret")));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn percent_encodes_secret_name_path_segment() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -53,7 +56,7 @@ async fn percent_encodes_secret_name_path_segment() {
|
|||
.await;
|
||||
|
||||
let secret = manager(&server)
|
||||
.get_secret_from_azure_key_vault("name/with spaces")
|
||||
.get_secret("name/with spaces")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
|
@ -61,9 +64,12 @@ async fn percent_encodes_secret_name_path_segment() {
|
|||
assert_eq!(secret.as_str(), Some("value"));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::not_found(404, None)]
|
||||
#[case::unauthorized(401, Some(401))]
|
||||
#[case::forbidden(403, Some(403))]
|
||||
#[case::throttled(429, Some(429))]
|
||||
#[case::server_error(500, Some(500))]
|
||||
#[tokio::test]
|
||||
async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option<u16>) {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -73,9 +79,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option<u
|
|||
.mount(&server)
|
||||
.await;
|
||||
|
||||
let result = manager(&server)
|
||||
.get_secret_from_azure_key_vault("NAME")
|
||||
.await;
|
||||
let result = manager(&server).get_secret("NAME").await;
|
||||
|
||||
match expected_status {
|
||||
None => assert_eq!(result.unwrap(), None),
|
||||
|
|
@ -83,6 +87,7 @@ async fn handles_statuses(#[case] status: u16, #[case] expected_status: Option<u
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn missing_value_is_an_error() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -93,41 +98,43 @@ async fn missing_value_is_an_error() {
|
|||
.await;
|
||||
|
||||
assert!(matches!(
|
||||
manager(&server)
|
||||
.get_secret_from_azure_key_vault("NAME")
|
||||
.await,
|
||||
manager(&server).get_secret("NAME").await,
|
||||
Err(Error::MissingValue)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_validates_vault_environment() {
|
||||
assert!(matches!(
|
||||
AzureKeyVault::new(Arc::new(|_: &str| None)),
|
||||
Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI"))
|
||||
));
|
||||
assert!(matches!(
|
||||
AzureKeyVault::new(Arc::new(|name: &str| {
|
||||
(name == "AZURE_KEY_VAULT_URI").then(|| "http://vault.example".to_owned())
|
||||
})),
|
||||
Err(Error::VaultUri)
|
||||
));
|
||||
assert!(matches!(
|
||||
AzureKeyVault::new(Arc::new(|name: &str| {
|
||||
(name == "AZURE_KEY_VAULT_URI").then(|| "vault.example".to_owned())
|
||||
})),
|
||||
Err(Error::VaultUri)
|
||||
));
|
||||
#[rstest]
|
||||
#[case::missing(None, true)]
|
||||
#[case::http(Some("http://vault.example"), false)]
|
||||
#[case::relative(Some("vault.example"), false)]
|
||||
#[case::malformed(Some("://"), false)]
|
||||
fn new_validates_vault_environment(
|
||||
#[case] uri: Option<&'static str>,
|
||||
#[case] missing_environment: bool,
|
||||
) {
|
||||
let result = AzureKeyVault::new(Arc::new(move |name: &str| {
|
||||
(name == "AZURE_KEY_VAULT_URI")
|
||||
.then(|| uri.map(str::to_owned))
|
||||
.flatten()
|
||||
}));
|
||||
|
||||
if missing_environment {
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI"))
|
||||
));
|
||||
} else {
|
||||
assert!(matches!(result, Err(Error::VaultUri)));
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case("https://myvault.vault.azure.net", "https://vault.azure.net/.default")]
|
||||
#[case(
|
||||
#[rstest]
|
||||
#[case::public_cloud("https://myvault.vault.azure.net", "https://vault.azure.net/.default")]
|
||||
#[case::government_cloud(
|
||||
"https://v.vault.usgovcloudapi.net/",
|
||||
"https://vault.usgovcloudapi.net/.default"
|
||||
)]
|
||||
#[case("http://localhost:8080", "https://localhost/.default")]
|
||||
#[test]
|
||||
#[case::local("http://localhost:8080", "https://localhost/.default")]
|
||||
fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) {
|
||||
let manager = AzureKeyVault::with_client(
|
||||
reqwest::Client::new(),
|
||||
|
|
@ -139,6 +146,7 @@ fn derives_scope_from_vault_host(#[case] uri: &str, #[case] expected: &str) {
|
|||
assert_eq!(manager.scope(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn missing_credentials_do_not_request_vault() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -150,7 +158,7 @@ async fn missing_credentials_do_not_request_vault() {
|
|||
|
||||
assert!(
|
||||
manager_without_credentials(&server)
|
||||
.get_secret_from_azure_key_vault("NAME")
|
||||
.get_secret("NAME")
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
|
|
@ -192,11 +200,15 @@ struct FixtureExpected {
|
|||
error: Option<bool>,
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn parity_fixture() -> Fixture {
|
||||
serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn parity_fixture_matches_python_backend_contract() {
|
||||
let fixture: Fixture =
|
||||
serde_json::from_str(include_str!("fixtures/key_vault_parity.json")).unwrap();
|
||||
for case in fixture.cases {
|
||||
async fn parity_fixture_matches_python_backend_contract(parity_fixture: Fixture) {
|
||||
for case in parity_fixture.cases {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(path(format!("/secrets/{}", case.secret_name)))
|
||||
.respond_with(
|
||||
|
|
@ -205,9 +217,7 @@ async fn parity_fixture_matches_python_backend_contract() {
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let result = manager(&server)
|
||||
.get_secret_from_azure_key_vault(&case.secret_name)
|
||||
.await;
|
||||
let result = manager(&server).get_secret(&case.secret_name).await;
|
||||
if case.expected.missing == Some(true) {
|
||||
assert_eq!(result.unwrap(), None);
|
||||
} else if case.expected.error == Some(true) {
|
||||
|
|
|
|||
|
|
@ -3,18 +3,16 @@ use std::sync::Arc;
|
|||
use litellm_core_utils::settings::ProcessEnvironment;
|
||||
use litellm_secrets_azure::AzureKeyVault;
|
||||
use litellm_secrets_types::Secret;
|
||||
use rstest::rstest;
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn reads_a_real_secret() {
|
||||
let environment = Arc::new(ProcessEnvironment);
|
||||
let manager = AzureKeyVault::new(environment).unwrap();
|
||||
let name = std::env::var("AZURE_KEY_VAULT_LIVE_SECRET_NAME").unwrap();
|
||||
let secret = manager
|
||||
.get_secret_from_azure_key_vault(&name)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let secret = manager.get_secret(&name).await.unwrap().unwrap();
|
||||
assert!(matches!(&secret, Secret::String(_)));
|
||||
let host = std::env::var("AZURE_KEY_VAULT_URI")
|
||||
.unwrap()
|
||||
|
|
|
|||
|
|
@ -2,7 +2,10 @@ use std::{fs, sync::Arc, time::Duration};
|
|||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secrets_types::{BaseSecretManager, SecretValue, validate_secret_name};
|
||||
use litellm_secrets_types::{
|
||||
BaseSecretManager, SecretOperationContext, SecretValue, SecretWriteContext,
|
||||
validate_secret_name,
|
||||
};
|
||||
use moka::future::Cache;
|
||||
use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode};
|
||||
|
||||
|
|
@ -140,7 +143,7 @@ impl CyberArkSecretManager {
|
|||
.map_err(|_| Error::Endpoint)
|
||||
}
|
||||
|
||||
async fn authenticate(&self) -> Result<SecretValue, Error> {
|
||||
async fn authenticate(&self, context: &SecretOperationContext) -> Result<SecretValue, Error> {
|
||||
if let Some(token) = self.token.get(&()).await {
|
||||
return Ok(token);
|
||||
}
|
||||
|
|
@ -155,12 +158,12 @@ impl CyberArkSecretManager {
|
|||
self.account, self.username
|
||||
))
|
||||
.map_err(|_| Error::Endpoint)?;
|
||||
let response = self
|
||||
.client
|
||||
.post(url)
|
||||
.body(self.api_key.expose().to_owned())
|
||||
.send()
|
||||
.await?;
|
||||
let response = with_timeout(
|
||||
self.client.post(url).body(self.api_key.expose().to_owned()),
|
||||
context,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(Error::AuthStatus(response.status().as_u16()));
|
||||
}
|
||||
|
|
@ -169,23 +172,37 @@ impl CyberArkSecretManager {
|
|||
Ok(token)
|
||||
}
|
||||
|
||||
async fn authorization_header(&self) -> Result<String, Error> {
|
||||
async fn authorization_header(
|
||||
&self,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<String, Error> {
|
||||
Ok(format!(
|
||||
"Token token=\"{}\"",
|
||||
self.authenticate().await?.expose()
|
||||
self.authenticate(context).await?.expose()
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
|
||||
self.async_read_secret_with_context(name, &SecretOperationContext::default())
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn async_read_secret_with_context(
|
||||
&self,
|
||||
name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<SecretValue>, Error> {
|
||||
if let Some(value) = self.secrets.get(name).await {
|
||||
return Ok(Some(value));
|
||||
}
|
||||
let response = self
|
||||
.client
|
||||
.get(self.secret_url(name)?)
|
||||
.header("Authorization", self.authorization_header().await?)
|
||||
.send()
|
||||
.await?;
|
||||
let response = with_timeout(
|
||||
self.client
|
||||
.get(self.secret_url(name)?)
|
||||
.header("Authorization", self.authorization_header(context).await?),
|
||||
context,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
if response.status() == reqwest::StatusCode::NOT_FOUND {
|
||||
return Ok(None);
|
||||
}
|
||||
|
|
@ -198,20 +215,38 @@ impl CyberArkSecretManager {
|
|||
}
|
||||
|
||||
pub async fn async_write_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
) -> Result<(), Error> {
|
||||
self.async_write_secret_with_context(
|
||||
name,
|
||||
value,
|
||||
description,
|
||||
&SecretOperationContext::default(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn async_write_secret_with_context(
|
||||
&self,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
_description: Option<&str>,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<(), Error> {
|
||||
validate_secret_name(name)?;
|
||||
self.ensure_variable_exists(name).await;
|
||||
let response = self
|
||||
.client
|
||||
.post(self.secret_url(name)?)
|
||||
.header("Authorization", self.authorization_header().await?)
|
||||
.body(value.expose().to_owned())
|
||||
.send()
|
||||
.await?;
|
||||
self.ensure_variable_exists(name, context).await;
|
||||
let response = with_timeout(
|
||||
self.client
|
||||
.post(self.secret_url(name)?)
|
||||
.header("Authorization", self.authorization_header(context).await?)
|
||||
.body(value.expose().to_owned()),
|
||||
context,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(Error::Status(response.status().as_u16()));
|
||||
}
|
||||
|
|
@ -219,7 +254,7 @@ impl CyberArkSecretManager {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_variable_exists(&self, name: &str) {
|
||||
async fn ensure_variable_exists(&self, name: &str, context: &SecretOperationContext) {
|
||||
let policy_url = self
|
||||
.endpoint
|
||||
.join(&format!("policies/{}/policy/root", self.account));
|
||||
|
|
@ -227,7 +262,7 @@ impl CyberArkSecretManager {
|
|||
tracing::warn!("Could not build CyberArk policy endpoint");
|
||||
return;
|
||||
};
|
||||
let Ok(authorization) = self.authorization_header().await else {
|
||||
let Ok(authorization) = self.authorization_header(context).await else {
|
||||
tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
|
||||
return;
|
||||
};
|
||||
|
|
@ -235,14 +270,16 @@ impl CyberArkSecretManager {
|
|||
"- !variable {}\n",
|
||||
serde_json::to_string(name).expect("serializing a string cannot fail")
|
||||
);
|
||||
let response = self
|
||||
.client
|
||||
.post(policy_url)
|
||||
.header("Authorization", authorization)
|
||||
.header("Content-Type", "application/x-yaml")
|
||||
.body(body)
|
||||
.send()
|
||||
.await;
|
||||
let response = with_timeout(
|
||||
self.client
|
||||
.post(policy_url)
|
||||
.header("Authorization", authorization)
|
||||
.header("Content-Type", "application/x-yaml")
|
||||
.body(body),
|
||||
context,
|
||||
)
|
||||
.send()
|
||||
.await;
|
||||
match response {
|
||||
Ok(response) if response.status().is_success() => {}
|
||||
Ok(response)
|
||||
|
|
@ -271,7 +308,21 @@ impl CyberArkSecretManager {
|
|||
pub async fn async_delete_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
_recovery_window_in_days: i64,
|
||||
recovery_window_in_days: Option<u32>,
|
||||
) -> Result<DeleteOutcome, Error> {
|
||||
self.async_delete_secret_with_context(
|
||||
name,
|
||||
recovery_window_in_days,
|
||||
&SecretOperationContext::default(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn async_delete_secret_with_context(
|
||||
&self,
|
||||
name: &str,
|
||||
_recovery_window_in_days: Option<u32>,
|
||||
_context: &SecretOperationContext,
|
||||
) -> Result<DeleteOutcome, Error> {
|
||||
tracing::warn!(
|
||||
"CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
|
||||
|
|
@ -286,29 +337,50 @@ impl BaseSecretManager for CyberArkSecretManager {
|
|||
type WriteResponse = ();
|
||||
type DeleteResponse = DeleteOutcome;
|
||||
|
||||
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
|
||||
self.async_read_secret(name).await
|
||||
async fn async_read_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<SecretValue>, Error> {
|
||||
self.async_read_secret_with_context(name, context).await
|
||||
}
|
||||
|
||||
async fn async_write_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
context: &SecretWriteContext,
|
||||
) -> Result<(), Error> {
|
||||
self.async_write_secret(name, value, description).await
|
||||
self.async_write_secret_with_context(
|
||||
name,
|
||||
value,
|
||||
context.description.as_deref(),
|
||||
&context.operation,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn async_delete_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
recovery_window_in_days: i64,
|
||||
recovery_window_in_days: Option<u32>,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<DeleteOutcome, Error> {
|
||||
self.async_delete_secret(name, recovery_window_in_days)
|
||||
self.async_delete_secret_with_context(name, recovery_window_in_days, context)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
fn with_timeout(
|
||||
request: reqwest::RequestBuilder,
|
||||
context: &SecretOperationContext,
|
||||
) -> reqwest::RequestBuilder {
|
||||
match context.timeout() {
|
||||
Some(timeout) => request.timeout(timeout),
|
||||
None => request,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_endpoint(mut endpoint: reqwest::Url) -> reqwest::Url {
|
||||
if !endpoint.path().ends_with('/') {
|
||||
endpoint.set_path(&format!("{}/", endpoint.path()));
|
||||
|
|
|
|||
|
|
@ -2,7 +2,10 @@ use std::{sync::Arc, time::Duration};
|
|||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use litellm_secrets_cyberark::{CyberArkSecretManager, DeleteOutcome, Error};
|
||||
use litellm_secrets_types::SecretValue;
|
||||
use litellm_secrets_types::{
|
||||
BaseSecretManager, CyberarkOperationContext, SecretOperationContext, SecretValue,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde::Deserialize;
|
||||
use wiremock::{
|
||||
Match, Mock, MockServer, Request, ResponseTemplate,
|
||||
|
|
@ -40,7 +43,8 @@ impl Match for RawPath {
|
|||
}
|
||||
}
|
||||
|
||||
fn fixture() -> ParityFixture {
|
||||
#[fixture]
|
||||
fn parity_fixture() -> ParityFixture {
|
||||
serde_json::from_str(include_str!("fixtures/parity.json")).unwrap()
|
||||
}
|
||||
|
||||
|
|
@ -65,6 +69,7 @@ async fn mount_auth(server: &MockServer, expected: u64) {
|
|||
.await;
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn successful_reads_cache_auth_secret_and_redact_values() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -89,6 +94,7 @@ async fn successful_reads_cache_auth_secret_and_redact_values() {
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn concurrent_reads_share_authentication_request() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -122,7 +128,7 @@ async fn concurrent_reads_share_authentication_request() {
|
|||
assert_eq!(second.unwrap().unwrap().expose(), "value");
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::not_found(404)]
|
||||
#[case::unauthorized(401)]
|
||||
#[case::forbidden(403)]
|
||||
|
|
@ -162,6 +168,7 @@ async fn failed_reads_are_not_cached(#[case] status: u16) {
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn failed_authentication_is_not_cached_and_does_not_read_secret() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -199,6 +206,29 @@ async fn failed_authentication_is_not_cached_and_does_not_read_secret() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn trait_read_applies_cyberark_operation_timeout_to_authentication() {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(path("/authn/acct/admin/authenticate"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_body_string(TOKEN_JSON)
|
||||
.set_delay(Duration::from_millis(50)),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, Duration::from_secs(60));
|
||||
let context = SecretOperationContext::Cyberark(CyberarkOperationContext {
|
||||
timeout: Some(Duration::from_millis(10)),
|
||||
});
|
||||
|
||||
let result = BaseSecretManager::async_read_secret(&manager, "key", &context).await;
|
||||
|
||||
assert!(matches!(result, Err(Error::Http(error)) if error.is_timeout()));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn expired_tokens_and_secrets_are_fetched_again() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -215,13 +245,14 @@ async fn expired_tokens_and_secrets_are_fetched_again() {
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::plain("OPENAI_API_KEY")]
|
||||
#[case::path("team/app/key")]
|
||||
#[case::punctuation("a b+c.d-e_f~g")]
|
||||
#[case::quote("needs \"quote\"")]
|
||||
#[tokio::test]
|
||||
async fn secret_names_use_python_quote_encoding(
|
||||
#[values("OPENAI_API_KEY", "team/app/key", "a b+c.d-e_f~g", "needs \"quote\"")] name: &str,
|
||||
) {
|
||||
let fixture = fixture();
|
||||
let secret = fixture
|
||||
async fn secret_names_use_python_quote_encoding(parity_fixture: ParityFixture, #[case] name: &str) {
|
||||
let secret = parity_fixture
|
||||
.secrets
|
||||
.iter()
|
||||
.find(|secret| secret.name == name)
|
||||
|
|
@ -244,11 +275,11 @@ async fn secret_names_use_python_quote_encoding(
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case(201)]
|
||||
#[case(409)]
|
||||
#[case(422)]
|
||||
#[case(500)]
|
||||
#[rstest]
|
||||
#[case::created(201)]
|
||||
#[case::already_exists(409)]
|
||||
#[case::unprocessable(422)]
|
||||
#[case::server_error(500)]
|
||||
#[tokio::test]
|
||||
async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u16) {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -282,6 +313,7 @@ async fn writes_tolerate_policy_status_and_cache_value(#[case] policy_status: u1
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn failed_value_write_is_not_cached() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -319,13 +351,17 @@ async fn failed_value_write_is_not_cached() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::parent("../etc")]
|
||||
#[case::embedded_parent("team/../etc")]
|
||||
#[case::control("key\n")]
|
||||
#[tokio::test]
|
||||
async fn unsafe_names_fail_before_http_calls() {
|
||||
async fn unsafe_names_fail_before_http_calls(#[case] name: &str) {
|
||||
let server = MockServer::start().await;
|
||||
let manager = manager(&server, Duration::from_secs(60));
|
||||
assert!(matches!(
|
||||
manager
|
||||
.async_write_secret("../etc", &SecretValue::new("v"), None)
|
||||
.async_write_secret(name, &SecretValue::new("v"), None)
|
||||
.await,
|
||||
Err(Error::Operation(
|
||||
litellm_secrets_types::Error::UnsafeSecretName
|
||||
|
|
@ -333,6 +369,7 @@ async fn unsafe_names_fail_before_http_calls() {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn delete_invalidates_cache_and_reports_not_supported() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -353,7 +390,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() {
|
|||
"v"
|
||||
);
|
||||
assert_eq!(
|
||||
manager.async_delete_secret("key", 7).await.unwrap(),
|
||||
manager.async_delete_secret("key", Some(7)).await.unwrap(),
|
||||
DeleteOutcome::NotSupported
|
||||
);
|
||||
assert_eq!(
|
||||
|
|
@ -367,7 +404,7 @@ async fn delete_invalidates_cache_and_reports_not_supported() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[rstest]
|
||||
fn new_validates_credentials_before_license_and_configuration() {
|
||||
let empty: Arc<dyn litellm_core_utils::settings::Lookup + Send + Sync> =
|
||||
Arc::new(|_: &str| None);
|
||||
|
|
@ -413,6 +450,7 @@ fn new_validates_credentials_before_license_and_configuration() {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn new_reads_environment_defaults_end_to_end() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -446,7 +484,7 @@ async fn new_reads_environment_defaults_end_to_end() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[rstest]
|
||||
fn new_reports_missing_client_certificate_files() {
|
||||
assert!(matches!(
|
||||
CyberArkSecretManager::new(
|
||||
|
|
@ -461,6 +499,7 @@ fn new_reports_missing_client_certificate_files() {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn trailing_slash_endpoint_preserves_base_path() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -494,23 +533,25 @@ async fn trailing_slash_endpoint_preserves_base_path() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parity_fixture_matches_authentication_contract() {
|
||||
let fixture = fixture();
|
||||
assert_eq!(fixture.endpoint, "http://conjur.test:8080");
|
||||
assert_eq!(fixture.account, "acct");
|
||||
assert_eq!(fixture.username, "admin");
|
||||
assert_eq!(fixture.api_key, "k3y");
|
||||
assert_eq!(fixture.authenticate_path, "/authn/acct/admin/authenticate");
|
||||
assert_eq!(fixture.token_json, TOKEN_JSON);
|
||||
#[rstest]
|
||||
fn parity_fixture_matches_authentication_contract(parity_fixture: ParityFixture) {
|
||||
assert_eq!(parity_fixture.endpoint, "http://conjur.test:8080");
|
||||
assert_eq!(parity_fixture.account, "acct");
|
||||
assert_eq!(parity_fixture.username, "admin");
|
||||
assert_eq!(parity_fixture.api_key, "k3y");
|
||||
assert_eq!(
|
||||
fixture.authorization_header,
|
||||
parity_fixture.authenticate_path,
|
||||
"/authn/acct/admin/authenticate"
|
||||
);
|
||||
assert_eq!(parity_fixture.token_json, TOKEN_JSON);
|
||||
assert_eq!(
|
||||
parity_fixture.authorization_header,
|
||||
format!("Token token=\"{}\"", STANDARD.encode(TOKEN_JSON))
|
||||
);
|
||||
assert_eq!(fixture.policy_path, "/policies/acct/policy/root");
|
||||
assert_eq!(fixture.secrets.len(), 4);
|
||||
assert_eq!(parity_fixture.policy_path, "/policies/acct/policy/root");
|
||||
assert_eq!(parity_fixture.secrets.len(), 4);
|
||||
assert_eq!(
|
||||
fixture.secrets[1].policy_body,
|
||||
parity_fixture.secrets[1].policy_body,
|
||||
"- !variable \"team/app/key\"\n"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use google_cloud_kms_v1::client::KeyManagementService;
|
||||
use litellm_secrets_google::GoogleKms;
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secrets_google::{Error, GoogleKms, kms::validate_environment};
|
||||
use rstest::rstest;
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
matchers::{body_json, path},
|
||||
};
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn google_kms_decrypts_using_the_configured_resource() {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -35,15 +38,49 @@ async fn google_kms_decrypts_using_the_configured_resource() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unset(None)]
|
||||
#[case::disabled(Some(false))]
|
||||
#[tokio::test]
|
||||
async fn disabled_google_kms_loader_does_not_require_environment_configuration() {
|
||||
async fn disabled_google_kms_loader_does_not_require_environment_configuration(
|
||||
#[case] enabled: Option<bool>,
|
||||
) {
|
||||
use std::sync::Arc;
|
||||
for enabled in [None, Some(false)] {
|
||||
assert!(
|
||||
litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None))
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::credentials_missing(None, None, "GOOGLE_APPLICATION_CREDENTIALS")]
|
||||
#[case::resource_missing(Some("credentials"), None, "GOOGLE_KMS_RESOURCE_NAME")]
|
||||
fn enabled_google_kms_requires_all_environment_values(
|
||||
#[case] credentials: Option<&str>,
|
||||
#[case] resource: Option<&str>,
|
||||
#[case] missing: &'static str,
|
||||
) {
|
||||
let environment = move |name: &str| match name {
|
||||
"GOOGLE_APPLICATION_CREDENTIALS" => credentials.map(str::to_owned),
|
||||
"GOOGLE_KMS_RESOURCE_NAME" => resource.map(str::to_owned),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
assert!(matches!(
|
||||
validate_environment(&environment as &dyn Lookup),
|
||||
Err(Error::MissingEnvironment(name)) if name == missing
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn complete_google_kms_environment_is_valid() {
|
||||
let environment = |name: &str| match name {
|
||||
"GOOGLE_APPLICATION_CREDENTIALS" => Some("credentials".to_owned()),
|
||||
"GOOGLE_KMS_RESOURCE_NAME" => Some("resource".to_owned()),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
assert!(validate_environment(&environment as &dyn Lookup).is_ok());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ use std::{sync::Arc, time::Duration};
|
|||
|
||||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use litellm_secrets_google::{Error, GoogleSecretManager};
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
use wiremock::{
|
||||
Mock, MockServer, ResponseTemplate,
|
||||
|
|
@ -20,11 +21,17 @@ fn manager(server: &MockServer, always_read: bool, ttl: Duration) -> GoogleSecre
|
|||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[fixture]
|
||||
fn default_ttl() -> Duration {
|
||||
Duration::from_secs(60)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::nonempty("private-value")]
|
||||
#[case::empty("")]
|
||||
#[tokio::test]
|
||||
async fn successful_reads_use_auth_latest_version_and_cache_including_empty_values(
|
||||
default_ttl: Duration,
|
||||
#[case] value: &str,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
|
|
@ -39,7 +46,7 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, false, Duration::from_secs(60));
|
||||
let manager = manager(&server, false, default_ttl);
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
manager
|
||||
|
|
@ -54,21 +61,44 @@ async fn successful_reads_use_auth_latest_version_and_cache_including_empty_valu
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::not_found(404, serde_json::json!({}))]
|
||||
#[case::unauthorized(401, serde_json::json!({}))]
|
||||
#[case::forbidden(403, serde_json::json!({}))]
|
||||
#[case::throttled(429, serde_json::json!({}))]
|
||||
#[case::unavailable(503, serde_json::json!({}))]
|
||||
#[case::missing_payload(200, serde_json::json!({"payload":{}}))]
|
||||
#[case::invalid_base64(200, serde_json::json!({"payload":{"data":"%%%"}}))]
|
||||
enum ExpectedReadFailure {
|
||||
Missing,
|
||||
Status(u16),
|
||||
MissingPayload,
|
||||
Base64,
|
||||
Utf8,
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::not_found(404, serde_json::json!({}), ExpectedReadFailure::Missing)]
|
||||
#[case::unauthorized(401, serde_json::json!({}), ExpectedReadFailure::Status(401))]
|
||||
#[case::forbidden(403, serde_json::json!({}), ExpectedReadFailure::Status(403))]
|
||||
#[case::throttled(429, serde_json::json!({}), ExpectedReadFailure::Status(429))]
|
||||
#[case::unavailable(503, serde_json::json!({}), ExpectedReadFailure::Status(503))]
|
||||
#[case::missing_payload(
|
||||
200,
|
||||
serde_json::json!({"payload":{}}),
|
||||
ExpectedReadFailure::MissingPayload
|
||||
)]
|
||||
#[case::invalid_base64(
|
||||
200,
|
||||
serde_json::json!({"payload":{"data":"%%%"}}),
|
||||
ExpectedReadFailure::Base64
|
||||
)]
|
||||
#[case::invalid_utf8(
|
||||
200,
|
||||
serde_json::json!({"payload":{"data":STANDARD.encode([0xff])}}),
|
||||
ExpectedReadFailure::Utf8
|
||||
)]
|
||||
#[tokio::test]
|
||||
async fn failed_or_missing_reads_are_not_cached(
|
||||
default_ttl: Duration,
|
||||
#[case] status: u16,
|
||||
#[case] body: serde_json::Value,
|
||||
#[case] expected: ExpectedReadFailure,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
let manager = manager(&server, false, Duration::from_secs(60));
|
||||
let manager = manager(&server, false, default_ttl);
|
||||
let failing = Mock::given(path(
|
||||
"/v1/projects/project/secrets/key/versions/latest:access",
|
||||
))
|
||||
|
|
@ -77,13 +107,16 @@ async fn failed_or_missing_reads_are_not_cached(
|
|||
.mount_as_scoped(&server)
|
||||
.await;
|
||||
let result = manager.get_secret_from_google_secret_manager("key").await;
|
||||
match status {
|
||||
404 => assert_eq!(result.unwrap(), None),
|
||||
200 => assert!(matches!(
|
||||
result,
|
||||
Err(Error::MissingPayload | Error::Base64(_))
|
||||
)),
|
||||
status => assert!(matches!(result, Err(Error::Status(actual)) if actual == status)),
|
||||
match expected {
|
||||
ExpectedReadFailure::Missing => assert_eq!(result.unwrap(), None),
|
||||
ExpectedReadFailure::Status(expected) => {
|
||||
assert!(matches!(result, Err(Error::Status(actual)) if actual == expected));
|
||||
}
|
||||
ExpectedReadFailure::MissingPayload => {
|
||||
assert!(matches!(result, Err(Error::MissingPayload)));
|
||||
}
|
||||
ExpectedReadFailure::Base64 => assert!(matches!(result, Err(Error::Base64(_)))),
|
||||
ExpectedReadFailure::Utf8 => assert!(matches!(result, Err(Error::Utf8))),
|
||||
}
|
||||
drop(failing);
|
||||
Mock::given(path(
|
||||
|
|
@ -109,7 +142,7 @@ async fn failed_or_missing_reads_are_not_cached(
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::always_read(true, Duration::from_secs(60))]
|
||||
#[case::expired_cache(false, Duration::from_millis(1))]
|
||||
#[tokio::test]
|
||||
|
|
@ -141,7 +174,7 @@ async fn always_read_and_expired_cache_fetch_again(
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[rstest]
|
||||
fn google_manager_requires_host_license_and_project_configuration() {
|
||||
assert!(matches!(
|
||||
GoogleSecretManager::new(Arc::new(|_: &str| None), false),
|
||||
|
|
@ -155,13 +188,29 @@ fn google_manager_requires_host_license_and_project_configuration() {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case("true")]
|
||||
#[case("null")]
|
||||
#[case("\"text\"")]
|
||||
#[case("{\"key\":1}")]
|
||||
#[rstest]
|
||||
#[case::provider_specific("GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL")]
|
||||
#[case::shared("SECRET_MANAGER_REFRESH_INTERVAL")]
|
||||
fn google_manager_rejects_invalid_refresh_intervals(#[case] variable: &'static str) {
|
||||
let environment = Arc::new(move |name: &str| match name {
|
||||
"GOOGLE_SECRET_MANAGER_PROJECT_ID" => Some("project".to_owned()),
|
||||
name if name == variable => Some("not-a-number".to_owned()),
|
||||
_ => None,
|
||||
});
|
||||
|
||||
assert!(matches!(
|
||||
GoogleSecretManager::new(environment, true),
|
||||
Err(Error::RefreshInterval)
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::boolean("true")]
|
||||
#[case::null("null")]
|
||||
#[case::string("\"text\"")]
|
||||
#[case::object("{\"key\":1}")]
|
||||
#[tokio::test]
|
||||
async fn cache_preserves_raw_values(#[case] raw: &str) {
|
||||
async fn cache_preserves_raw_values(default_ttl: Duration, #[case] raw: &str) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(path(
|
||||
"/v1/projects/project/secrets/key/versions/latest:access",
|
||||
|
|
@ -173,7 +222,7 @@ async fn cache_preserves_raw_values(#[case] raw: &str) {
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, false, Duration::from_secs(60));
|
||||
let manager = manager(&server, false, default_ttl);
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
manager
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ pub enum Error {
|
|||
EnterpriseRequired,
|
||||
#[error("invalid secret name")]
|
||||
InvalidSecretName(#[from] litellm_secrets_types::Error),
|
||||
#[error("HashiCorp Vault received an incompatible operation context")]
|
||||
InvalidOperationContext,
|
||||
#[error("HashiCorp Vault client failed")]
|
||||
Client(
|
||||
#[from]
|
||||
|
|
@ -29,6 +31,8 @@ pub enum Error {
|
|||
MalformedPayload,
|
||||
#[error("HashiCorp Vault secret value is not a string")]
|
||||
NonStringValue,
|
||||
#[error("HashiCorp Vault operation timed out")]
|
||||
Timeout,
|
||||
#[error("invalid HashiCorp Vault refresh interval")]
|
||||
RefreshInterval,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
use std::{
|
||||
collections::HashMap,
|
||||
fmt,
|
||||
future::Future,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secrets_types::{
|
||||
BaseSecretManager, SecretValue, async_rotate_secret, validate_secret_name,
|
||||
BaseSecretManager, HashicorpOperationContext, SecretOperationContext, SecretValue,
|
||||
SecretWriteContext, async_rotate_secret, validate_secret_name,
|
||||
};
|
||||
use moka::future::Cache;
|
||||
use rustify::errors::ClientError as RustifyClientError;
|
||||
|
|
@ -31,17 +33,23 @@ struct CachedClient {
|
|||
expires_at: Option<Instant>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
|
||||
pub struct SecretLocation {
|
||||
pub namespace: Option<String>,
|
||||
pub mount: String,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Hash, PartialEq, Eq)]
|
||||
struct CacheKey {
|
||||
location: SecretLocation,
|
||||
data_key: String,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HashicorpVault {
|
||||
config: HashicorpVaultConfig,
|
||||
cache: Cache<String, SecretValue>,
|
||||
cache: Cache<CacheKey, SecretValue>,
|
||||
auth_client: Arc<Mutex<Option<CachedClient>>>,
|
||||
}
|
||||
|
||||
|
|
@ -71,7 +79,7 @@ impl HashicorpVault {
|
|||
if !enterprise_enabled {
|
||||
return Err(Error::EnterpriseRequired);
|
||||
}
|
||||
let cache: Cache<String, SecretValue> = Cache::builder()
|
||||
let cache: Cache<CacheKey, SecretValue> = Cache::builder()
|
||||
.max_capacity(CACHE_CAPACITY)
|
||||
.time_to_live(config.refresh_interval)
|
||||
.build();
|
||||
|
|
@ -83,9 +91,21 @@ impl HashicorpVault {
|
|||
}
|
||||
|
||||
pub fn secret_location(&self, secret_name: &str) -> Result<SecretLocation, Error> {
|
||||
self.secret_location_with_context(secret_name, &SecretOperationContext::default())
|
||||
}
|
||||
|
||||
pub fn secret_location_with_context(
|
||||
&self,
|
||||
secret_name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<SecretLocation, Error> {
|
||||
validate_secret_name(secret_name).map_err(Error::InvalidSecretName)?;
|
||||
let operation: Option<&HashicorpOperationContext> = hashicorp_context(context)?;
|
||||
let path: String = [
|
||||
self.config.path_prefix.clone(),
|
||||
operation
|
||||
.and_then(|operation| operation.path_prefix.as_deref())
|
||||
.and_then(path_component)
|
||||
.or_else(|| self.config.path_prefix.clone()),
|
||||
Some(secret_name.to_owned()),
|
||||
]
|
||||
.into_iter()
|
||||
|
|
@ -94,7 +114,10 @@ impl HashicorpVault {
|
|||
.join("/");
|
||||
Ok(SecretLocation {
|
||||
namespace: self.config.secret_namespace().map(str::to_owned),
|
||||
mount: self.config.mount.clone(),
|
||||
mount: operation
|
||||
.and_then(|operation| operation.mount.as_deref())
|
||||
.and_then(path_component)
|
||||
.unwrap_or_else(|| self.config.mount.clone()),
|
||||
path,
|
||||
})
|
||||
}
|
||||
|
|
@ -104,19 +127,37 @@ impl HashicorpVault {
|
|||
}
|
||||
|
||||
pub async fn async_read_secret(&self, secret_name: &str) -> Result<Option<SecretValue>, Error> {
|
||||
let location: SecretLocation = self.secret_location(secret_name)?;
|
||||
let cache_key: String = cache_key(&location);
|
||||
self.async_read_secret_with_context(secret_name, &SecretOperationContext::default())
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn async_read_secret_with_context(
|
||||
&self,
|
||||
secret_name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<SecretValue>, Error> {
|
||||
let location: SecretLocation = self.secret_location_with_context(secret_name, context)?;
|
||||
let data_key: String = data_key(context)?;
|
||||
let cache_key = CacheKey {
|
||||
location: location.clone(),
|
||||
data_key: data_key.clone(),
|
||||
};
|
||||
if let Some(value) = self.cache.get(&cache_key).await {
|
||||
return Ok(Some(value));
|
||||
}
|
||||
let client: Arc<VaultClient> = self.vault_client().await?;
|
||||
let data: HashMap<String, Value> =
|
||||
let data: Option<HashMap<String, Value>> = with_timeout(context, async {
|
||||
let client: Arc<VaultClient> = self.vault_client().await?;
|
||||
match kv2::read(client.as_ref(), &location.mount, &location.path).await {
|
||||
Ok(data) => data,
|
||||
Err(error) if api_status(&error) == Some(404) => return Ok(None),
|
||||
Err(error) => return Err(map_api_error(error, ErrorContext::Read)),
|
||||
};
|
||||
let Some(value) = data.get("key") else {
|
||||
Ok(data) => Ok(Some(data)),
|
||||
Err(error) if api_status(&error) == Some(404) => Ok(None),
|
||||
Err(error) => Err(map_api_error(error, ErrorContext::Read)),
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
let Some(data) = data else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(value) = data.get(&data_key) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let value: &str = value.as_str().ok_or(Error::NonStringValue)?;
|
||||
|
|
@ -131,11 +172,29 @@ impl HashicorpVault {
|
|||
value: SecretValue,
|
||||
description: Option<&str>,
|
||||
) -> Result<Value, Error> {
|
||||
let location: SecretLocation = self.secret_location(secret_name)?;
|
||||
let cache_key: String = cache_key(&location);
|
||||
let data: HashMap<String, Value> = match description {
|
||||
self.async_write_secret_with_context(
|
||||
secret_name,
|
||||
&value,
|
||||
&SecretWriteContext {
|
||||
description: description.map(str::to_owned),
|
||||
..SecretWriteContext::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn async_write_secret_with_context(
|
||||
&self,
|
||||
secret_name: &str,
|
||||
value: &SecretValue,
|
||||
context: &SecretWriteContext,
|
||||
) -> Result<Value, Error> {
|
||||
let location: SecretLocation =
|
||||
self.secret_location_with_context(secret_name, &context.operation)?;
|
||||
let data_key: String = data_key(&context.operation)?;
|
||||
let data: HashMap<String, Value> = match context.description.as_deref() {
|
||||
Some(description) => [
|
||||
("key".to_owned(), Value::String(value.expose().to_owned())),
|
||||
(data_key, Value::String(value.expose().to_owned())),
|
||||
(
|
||||
"description".to_owned(),
|
||||
Value::String(description.to_owned()),
|
||||
|
|
@ -143,27 +202,41 @@ impl HashicorpVault {
|
|||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
None => [("key".to_owned(), Value::String(value.expose().to_owned()))]
|
||||
None => [(data_key, Value::String(value.expose().to_owned()))]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
};
|
||||
let client: Arc<VaultClient> = self.vault_client().await?;
|
||||
let metadata = kv2::set(client.as_ref(), &location.mount, &location.path, &data)
|
||||
.await
|
||||
.map_err(|error| map_api_error(error, ErrorContext::Secret))?;
|
||||
self.cache.invalidate(&cache_key).await;
|
||||
let metadata = with_timeout(&context.operation, async {
|
||||
let client: Arc<VaultClient> = self.vault_client().await?;
|
||||
kv2::set(client.as_ref(), &location.mount, &location.path, &data)
|
||||
.await
|
||||
.map_err(|error| map_api_error(error, ErrorContext::Secret))
|
||||
})
|
||||
.await?;
|
||||
self.cache.invalidate_all();
|
||||
serde_json::to_value(metadata)
|
||||
.map_err(|source| Error::Client(ClientError::JsonParseError { source }))
|
||||
}
|
||||
|
||||
pub async fn async_delete_secret(&self, secret_name: &str) -> Result<(), Error> {
|
||||
let location: SecretLocation = self.secret_location(secret_name)?;
|
||||
let cache_key: String = cache_key(&location);
|
||||
let client: Arc<VaultClient> = self.vault_client().await?;
|
||||
kv2::delete_latest(client.as_ref(), &location.mount, &location.path)
|
||||
self.async_delete_secret_with_context(secret_name, &SecretOperationContext::default())
|
||||
.await
|
||||
.map_err(|error| map_api_error(error, ErrorContext::Secret))?;
|
||||
self.cache.invalidate(&cache_key).await;
|
||||
}
|
||||
|
||||
pub async fn async_delete_secret_with_context(
|
||||
&self,
|
||||
secret_name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<(), Error> {
|
||||
let location: SecretLocation = self.secret_location_with_context(secret_name, context)?;
|
||||
with_timeout(context, async {
|
||||
let client: Arc<VaultClient> = self.vault_client().await?;
|
||||
kv2::delete_latest(client.as_ref(), &location.mount, &location.path)
|
||||
.await
|
||||
.map_err(|error| map_api_error(error, ErrorContext::Secret))
|
||||
})
|
||||
.await?;
|
||||
self.cache.invalidate_all();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -173,7 +246,23 @@ impl HashicorpVault {
|
|||
new_name: &str,
|
||||
value: &SecretValue,
|
||||
) -> Result<Value, Error> {
|
||||
async_rotate_secret(self, current_name, new_name, value).await
|
||||
self.async_rotate_secret_with_context(
|
||||
current_name,
|
||||
new_name,
|
||||
value,
|
||||
&SecretOperationContext::default(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn async_rotate_secret_with_context(
|
||||
&self,
|
||||
current_name: &str,
|
||||
new_name: &str,
|
||||
value: &SecretValue,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Value, Error> {
|
||||
async_rotate_secret(self, current_name, new_name, value, context).await
|
||||
}
|
||||
|
||||
async fn vault_client(&self) -> Result<Arc<VaultClient>, Error> {
|
||||
|
|
@ -254,25 +343,30 @@ impl BaseSecretManager for HashicorpVault {
|
|||
type WriteResponse = Value;
|
||||
type DeleteResponse = ();
|
||||
|
||||
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
|
||||
HashicorpVault::async_read_secret(self, name).await
|
||||
async fn async_read_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<SecretValue>, Error> {
|
||||
HashicorpVault::async_read_secret_with_context(self, name, context).await
|
||||
}
|
||||
|
||||
async fn async_write_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
context: &SecretWriteContext,
|
||||
) -> Result<Value, Error> {
|
||||
HashicorpVault::async_write_secret(self, name, value.clone(), description).await
|
||||
HashicorpVault::async_write_secret_with_context(self, name, value, context).await
|
||||
}
|
||||
|
||||
async fn async_delete_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
_recovery_window_in_days: i64,
|
||||
_recovery_window_in_days: Option<u32>,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<(), Error> {
|
||||
HashicorpVault::async_delete_secret(self, name).await
|
||||
HashicorpVault::async_delete_secret_with_context(self, name, context).await
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -283,11 +377,42 @@ enum ErrorContext {
|
|||
Secret,
|
||||
}
|
||||
|
||||
fn cache_key(location: &SecretLocation) -> String {
|
||||
format!(
|
||||
"{:?}/{}/{}",
|
||||
location.namespace, location.mount, location.path
|
||||
)
|
||||
fn hashicorp_context(
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<&HashicorpOperationContext>, Error> {
|
||||
match context {
|
||||
SecretOperationContext::Hashicorp(context) => Ok(Some(context)),
|
||||
SecretOperationContext::Default => Ok(None),
|
||||
SecretOperationContext::Aws(_) | SecretOperationContext::Cyberark(_) => {
|
||||
Err(Error::InvalidOperationContext)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn path_component(value: &str) -> Option<String> {
|
||||
let value: &str = value.trim().trim_matches('/');
|
||||
(!value.is_empty()).then(|| value.to_owned())
|
||||
}
|
||||
|
||||
fn data_key(context: &SecretOperationContext) -> Result<String, Error> {
|
||||
Ok(hashicorp_context(context)?
|
||||
.and_then(|context| context.data_key.as_deref())
|
||||
.map(str::trim)
|
||||
.filter(|data_key| !data_key.is_empty())
|
||||
.map(str::to_owned)
|
||||
.unwrap_or_else(|| "key".to_owned()))
|
||||
}
|
||||
|
||||
async fn with_timeout<T>(
|
||||
context: &SecretOperationContext,
|
||||
operation: impl Future<Output = Result<T, Error>>,
|
||||
) -> Result<T, Error> {
|
||||
match context.timeout() {
|
||||
Some(timeout) => tokio::time::timeout(timeout, operation)
|
||||
.await
|
||||
.map_err(|_| Error::Timeout)?,
|
||||
None => operation.await,
|
||||
}
|
||||
}
|
||||
|
||||
fn identity_for(tls: Option<&TlsCertAuth>) -> Result<Option<Identity>, Error> {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
|
|||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
use litellm_secrets_hashicorp::{Error, HashicorpVault, HashicorpVaultConfig};
|
||||
use litellm_secrets_types::SecretValue;
|
||||
use litellm_secrets_types::{
|
||||
AwsOperationContext, BaseSecretManager, CyberarkOperationContext, HashicorpOperationContext,
|
||||
SecretOperationContext, SecretValue, SecretWriteContext,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use wiremock::{
|
||||
|
|
@ -69,8 +73,14 @@ fn read_response(data: serde_json::Value) -> serde_json::Value {
|
|||
})
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn token_values() -> Vec<(&'static str, &'static str)> {
|
||||
vec![("HCP_VAULT_TOKEN", "token")]
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn token_reads_use_vault_headers_and_cache_values() {
|
||||
async fn token_reads_use_vault_headers_and_cache_values(token_values: Vec<(&str, &str)>) {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/secret/data/name"))
|
||||
|
|
@ -81,7 +91,7 @@ async fn token_reads_use_vault_headers_and_cache_values() {
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]);
|
||||
let manager: HashicorpVault = manager(&server, &token_values);
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
|
|
@ -109,6 +119,7 @@ async fn token_reads_use_vault_headers_and_cache_values() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
|
|
@ -138,7 +149,7 @@ async fn namespace_mount_and_prefix_are_sanitized_in_the_url() {
|
|||
assert!(manager.async_read_secret("name").await.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[rstest]
|
||||
fn trailing_address_slashes_are_removed() {
|
||||
let environment: Arc<dyn Lookup + Send + Sync> = Arc::new(|name: &str| match name {
|
||||
"HCP_VAULT_ADDR" => Some("http://vault.test:8200///".to_owned()),
|
||||
|
|
@ -159,9 +170,9 @@ fn trailing_address_slashes_are_removed() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case("-1")]
|
||||
#[case("not-a-number")]
|
||||
#[rstest]
|
||||
#[case::negative("-1")]
|
||||
#[case::not_a_number("not-a-number")]
|
||||
fn invalid_refresh_intervals_are_rejected(#[case] value: &str) {
|
||||
let environment: Arc<dyn Lookup + Send + Sync> = Arc::new(move |name: &str| match name {
|
||||
"HCP_VAULT_REFRESH_INTERVAL" => Some(value.to_owned()),
|
||||
|
|
@ -174,6 +185,7 @@ fn invalid_refresh_intervals_are_rejected(#[case] value: &str) {
|
|||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn approle_login_uses_namespace_and_reuses_the_token() {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
|
|
@ -216,6 +228,7 @@ async fn approle_login_uses_namespace_and_reuses_the_token() {
|
|||
assert!(manager.async_read_secret("name-2").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn approle_tokens_expire_after_the_vault_lease() {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
|
|
@ -246,6 +259,7 @@ async fn approle_tokens_expire_after_the_vault_lease() {
|
|||
assert!(manager.async_read_secret("second").await.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn tls_login_posts_the_role_and_uses_the_client_identity() {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
|
|
@ -343,21 +357,29 @@ async fn tls_login_posts_the_role_and_uses_the_client_identity() {
|
|||
assert!(login_bodies.contains(&json!({})));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::missing(404, json!({"errors": ["missing"]}), 0)]
|
||||
#[case::malformed(200, json!({"data": "invalid"}), 1)]
|
||||
#[case::missing_key(200, json!({}), 0)]
|
||||
#[case::non_string(200, json!({"key": 1}), 2)]
|
||||
#[derive(Clone, Copy)]
|
||||
enum ExpectedRead {
|
||||
Missing,
|
||||
Malformed,
|
||||
NonString,
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing(404, json!({"errors": ["missing"]}), ExpectedRead::Missing)]
|
||||
#[case::malformed(200, json!({"data": "invalid"}), ExpectedRead::Malformed)]
|
||||
#[case::missing_key(200, json!({}), ExpectedRead::Missing)]
|
||||
#[case::non_string(200, json!({"key": 1}), ExpectedRead::NonString)]
|
||||
#[tokio::test]
|
||||
async fn read_responses_distinguish_absence_and_malformed_payloads(
|
||||
token_values: Vec<(&str, &str)>,
|
||||
#[case] status: u16,
|
||||
#[case] body: serde_json::Value,
|
||||
#[case] expected: u8,
|
||||
#[case] expected: ExpectedRead,
|
||||
) {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.respond_with(ResponseTemplate::new(status).set_body_json(
|
||||
if status == 200 && expected != 1 {
|
||||
if status == 200 && !matches!(expected, ExpectedRead::Malformed) {
|
||||
read_response(body)
|
||||
} else {
|
||||
body
|
||||
|
|
@ -366,20 +388,19 @@ async fn read_responses_distinguish_absence_and_malformed_payloads(
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let result: Result<Option<SecretValue>, Error> =
|
||||
manager(&server, &[("HCP_VAULT_TOKEN", "token")])
|
||||
.async_read_secret("name")
|
||||
.await;
|
||||
let result: Result<Option<SecretValue>, Error> = manager(&server, &token_values)
|
||||
.async_read_secret("name")
|
||||
.await;
|
||||
match expected {
|
||||
0 => assert!(result.unwrap().is_none()),
|
||||
1 => assert!(matches!(result, Err(Error::MalformedPayload))),
|
||||
2 => assert!(matches!(result, Err(Error::NonStringValue))),
|
||||
_ => unreachable!(),
|
||||
ExpectedRead::Missing => assert!(result.unwrap().is_none()),
|
||||
ExpectedRead::Malformed => assert!(matches!(result, Err(Error::MalformedPayload))),
|
||||
ExpectedRead::NonString => assert!(matches!(result, Err(Error::NonStringValue))),
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn write_and_delete_invalidate_the_read_cache() {
|
||||
async fn write_and_delete_invalidate_the_read_cache(token_values: Vec<(&str, &str)>) {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/secret/data/name"))
|
||||
|
|
@ -418,7 +439,7 @@ async fn write_and_delete_invalidate_the_read_cache() {
|
|||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager: HashicorpVault = manager(&server, &[("HCP_VAULT_TOKEN", "token")]);
|
||||
let manager: HashicorpVault = manager(&server, &token_values);
|
||||
|
||||
assert!(manager.async_read_secret("name").await.unwrap().is_some());
|
||||
assert!(
|
||||
|
|
@ -431,6 +452,313 @@ async fn write_and_delete_invalidate_the_read_cache() {
|
|||
manager.async_delete_secret("name").await.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn base_manager_context_overrides_vault_location_and_data_key(
|
||||
token_values: Vec<(&str, &str)>,
|
||||
) {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/alternate/data/managed/name"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(read_response(json!({"api_token": "value"}))),
|
||||
)
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/alternate/data/managed/name"))
|
||||
.and(body_json(json!({
|
||||
"data": {"api_token": "updated", "description": "Managed key"}
|
||||
})))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(json!({
|
||||
"data": {
|
||||
"created_time": "",
|
||||
"deletion_time": "",
|
||||
"custom_metadata": null,
|
||||
"destroyed": false,
|
||||
"version": 2
|
||||
},
|
||||
"lease_id": "",
|
||||
"lease_duration": 0,
|
||||
"renewable": false,
|
||||
"request_id": "",
|
||||
"warnings": null,
|
||||
"wrap_info": null
|
||||
})))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("DELETE"))
|
||||
.and(path("/v1/alternate/data/managed/name"))
|
||||
.respond_with(ResponseTemplate::new(204))
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager: HashicorpVault = manager(&server, &token_values);
|
||||
let operation = SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
mount: Some(" /alternate/ ".to_owned()),
|
||||
path_prefix: Some(" /managed/ ".to_owned()),
|
||||
data_key: Some("api_token".to_owned()),
|
||||
..HashicorpOperationContext::default()
|
||||
});
|
||||
let write_context = SecretWriteContext {
|
||||
description: Some("Managed key".to_owned()),
|
||||
operation: operation.clone(),
|
||||
..SecretWriteContext::default()
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
BaseSecretManager::async_read_secret(&manager, "name", &operation)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
"value"
|
||||
);
|
||||
BaseSecretManager::async_write_secret(
|
||||
&manager,
|
||||
"name",
|
||||
&SecretValue::new("updated"),
|
||||
&write_context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
BaseSecretManager::async_read_secret(&manager, "name", &operation)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
BaseSecretManager::async_delete_secret(&manager, "name", None, &operation)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn reads_cache_each_data_key_for_the_same_vault_path(token_values: Vec<(&str, &str)>) {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/secret/data/name"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(read_response(json!({
|
||||
"key": "primary",
|
||||
"alternate": "secondary"
|
||||
}))),
|
||||
)
|
||||
.expect(2)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager: HashicorpVault = manager(&server, &token_values);
|
||||
let alternate = SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
data_key: Some("alternate".to_owned()),
|
||||
..HashicorpOperationContext::default()
|
||||
});
|
||||
|
||||
assert_eq!(
|
||||
manager
|
||||
.async_read_secret("name")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
"primary"
|
||||
);
|
||||
assert_eq!(
|
||||
BaseSecretManager::async_read_secret(&manager, "name", &alternate)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
"secondary"
|
||||
);
|
||||
assert_eq!(
|
||||
manager
|
||||
.async_read_secret("name")
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.expose(),
|
||||
"primary"
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn base_manager_context_timeout_limits_vault_io(token_values: Vec<(&str, &str)>) {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/secret/data/name"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_delay(Duration::from_millis(100))
|
||||
.set_body_json(read_response(json!({"key": "value"}))),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager: HashicorpVault = manager(&server, &token_values);
|
||||
let context = SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
timeout: Some(Duration::from_millis(10)),
|
||||
..HashicorpOperationContext::default()
|
||||
});
|
||||
|
||||
assert!(matches!(
|
||||
BaseSecretManager::async_read_secret(&manager, "name", &context).await,
|
||||
Err(Error::Timeout)
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::aws(SecretOperationContext::Aws(AwsOperationContext::default()))]
|
||||
#[case::cyberark(SecretOperationContext::Cyberark(CyberarkOperationContext::default()))]
|
||||
#[tokio::test]
|
||||
async fn foreign_contexts_cannot_access_vault_secrets(
|
||||
token_values: Vec<(&str, &str)>,
|
||||
#[case] context: SecretOperationContext,
|
||||
#[values(false, true)] cached: bool,
|
||||
) {
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/secret/data/name"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(read_response(json!({"key": "value"}))),
|
||||
)
|
||||
.expect(u64::from(cached))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, &token_values);
|
||||
if cached {
|
||||
assert!(manager.async_read_secret("name").await.unwrap().is_some());
|
||||
}
|
||||
assert!(matches!(
|
||||
BaseSecretManager::async_read_secret(&manager, "name", &context).await,
|
||||
Err(Error::InvalidOperationContext)
|
||||
));
|
||||
assert!(matches!(
|
||||
BaseSecretManager::async_write_secret(
|
||||
&manager,
|
||||
"name",
|
||||
&SecretValue::new("replacement"),
|
||||
&SecretWriteContext {
|
||||
operation: context.clone(),
|
||||
..SecretWriteContext::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
Err(Error::InvalidOperationContext)
|
||||
));
|
||||
assert!(matches!(
|
||||
BaseSecretManager::async_delete_secret(&manager, "name", None, &context).await,
|
||||
Err(Error::InvalidOperationContext)
|
||||
));
|
||||
assert!(matches!(
|
||||
manager
|
||||
.async_rotate_secret_with_context(
|
||||
"name",
|
||||
"new",
|
||||
&SecretValue::new("replacement"),
|
||||
&context
|
||||
)
|
||||
.await,
|
||||
Err(Error::InvalidOperationContext)
|
||||
));
|
||||
assert_eq!(
|
||||
server.received_requests().await.unwrap().len(),
|
||||
usize::from(cached)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rotation_applies_timeout_to_each_request(token_values: Vec<(&str, &str)>) {
|
||||
let server = MockServer::start().await;
|
||||
let timeout = Duration::from_secs(1);
|
||||
let delay = timeout / 2;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/alternate/data/managed/current"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_delay(delay)
|
||||
.set_body_json(read_response(json!({"api_token": "original"}))),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.and(path("/v1/alternate/data/managed/new"))
|
||||
.and(body_json(json!({
|
||||
"data": {"api_token": "replacement", "description": "Rotated from current"}
|
||||
})))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_delay(delay)
|
||||
.set_body_json(json!({
|
||||
"data": {
|
||||
"created_time": "",
|
||||
"deletion_time": "",
|
||||
"custom_metadata": null,
|
||||
"destroyed": false,
|
||||
"version": 1
|
||||
},
|
||||
"lease_id": "",
|
||||
"lease_duration": 0,
|
||||
"renewable": false,
|
||||
"request_id": "",
|
||||
"warnings": null,
|
||||
"wrap_info": null
|
||||
})),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/v1/alternate/data/managed/new"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200)
|
||||
.set_delay(delay)
|
||||
.set_body_json(read_response(json!({"api_token": "replacement"}))),
|
||||
)
|
||||
.mount(&server)
|
||||
.await;
|
||||
Mock::given(method("DELETE"))
|
||||
.and(path("/v1/alternate/data/managed/current"))
|
||||
.respond_with(ResponseTemplate::new(204).set_delay(delay))
|
||||
.mount(&server)
|
||||
.await;
|
||||
let manager = manager(&server, &token_values);
|
||||
let context = SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
timeout: Some(timeout),
|
||||
mount: Some("alternate".to_owned()),
|
||||
path_prefix: Some("managed".to_owned()),
|
||||
data_key: Some("api_token".to_owned()),
|
||||
});
|
||||
|
||||
manager
|
||||
.async_rotate_secret_with_context(
|
||||
"current",
|
||||
"new",
|
||||
&SecretValue::new("replacement"),
|
||||
&context,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let requests = server.received_requests().await.unwrap();
|
||||
let operations: Vec<_> = requests
|
||||
.iter()
|
||||
.map(|request| (request.method.as_str(), request.url.path()))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
operations,
|
||||
[
|
||||
("GET", "/v1/alternate/data/managed/current"),
|
||||
("POST", "/v1/alternate/data/managed/new"),
|
||||
("GET", "/v1/alternate/data/managed/new"),
|
||||
("DELETE", "/v1/alternate/data/managed/current"),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn no_auth_and_invalid_names_fail_without_requests() {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
|
|
@ -447,6 +775,7 @@ async fn no_auth_and_invalid_names_fail_without_requests() {
|
|||
assert!(server.received_requests().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn debug_output_redacts_authentication_values() {
|
||||
let server: MockServer = MockServer::start().await;
|
||||
|
|
@ -468,14 +797,18 @@ struct ParityCase {
|
|||
secret_name: String,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configuration_matches_python_parity_fixture() {
|
||||
let cases: Vec<ParityCase> = serde_json::from_str(include_str!(concat!(
|
||||
#[fixture]
|
||||
fn parity_cases() -> Vec<ParityCase> {
|
||||
serde_json::from_str(include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../tests/test_litellm/secret_managers/hashicorp_vault_parity.json"
|
||||
)))
|
||||
.unwrap();
|
||||
for case in cases {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn configuration_matches_python_parity_fixture(parity_cases: Vec<ParityCase>) {
|
||||
for case in parity_cases {
|
||||
let values: HashMap<String, String> = case.env.clone();
|
||||
let environment: Arc<dyn Lookup + Send + Sync> =
|
||||
Arc::new(move |name: &str| values.get(name).cloned());
|
||||
|
|
@ -521,6 +854,7 @@ fn configuration_matches_python_parity_fixture() {
|
|||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn live_vault_round_trip() {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use crate::{Error, SecretValue};
|
||||
use crate::{Error, SecretOperationContext, SecretValue, SecretWriteContext};
|
||||
|
||||
pub fn validate_secret_name(name: &str) -> Result<(), Error> {
|
||||
if name.split('/').any(|segment| segment == "..")
|
||||
|
|
@ -20,17 +20,22 @@ pub trait BaseSecretManager {
|
|||
type WriteResponse;
|
||||
type DeleteResponse;
|
||||
|
||||
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Self::Error>;
|
||||
async fn async_read_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<SecretValue>, Self::Error>;
|
||||
async fn async_write_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
context: &SecretWriteContext,
|
||||
) -> Result<Self::WriteResponse, Self::Error>;
|
||||
async fn async_delete_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
recovery_window_in_days: i64,
|
||||
recovery_window_in_days: Option<u32>,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Self::DeleteResponse, Self::Error>;
|
||||
}
|
||||
|
||||
|
|
@ -39,20 +44,31 @@ pub async fn async_rotate_secret<M: BaseSecretManager>(
|
|||
current_name: &str,
|
||||
new_name: &str,
|
||||
value: &SecretValue,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<M::WriteResponse, M::Error> {
|
||||
if manager.async_read_secret(current_name).await?.is_none() {
|
||||
if manager
|
||||
.async_read_secret(current_name, context)
|
||||
.await?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::CurrentSecretMissing.into());
|
||||
}
|
||||
let response = manager
|
||||
.async_write_secret(
|
||||
new_name,
|
||||
value,
|
||||
Some(&format!("Rotated from {current_name}")),
|
||||
&SecretWriteContext::rotated_from(current_name, context.clone()),
|
||||
)
|
||||
.await?;
|
||||
if manager.async_read_secret(new_name).await?.is_none() {
|
||||
if manager
|
||||
.async_read_secret(new_name, context)
|
||||
.await?
|
||||
.is_none()
|
||||
{
|
||||
return Err(Error::NewSecretMissing.into());
|
||||
}
|
||||
manager.async_delete_secret(current_name, 7).await?;
|
||||
manager
|
||||
.async_delete_secret(current_name, Some(7), context)
|
||||
.await?;
|
||||
Ok(response)
|
||||
}
|
||||
|
|
|
|||
65
litellm-rust/crates/secrets-types/src/context.rs
Normal file
65
litellm-rust/crates/secrets-types/src/context.rs
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
use std::{collections::BTreeMap, time::Duration};
|
||||
|
||||
use crate::SecretValue;
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub enum SecretOperationContext {
|
||||
#[default]
|
||||
Default,
|
||||
Aws(AwsOperationContext),
|
||||
Hashicorp(HashicorpOperationContext),
|
||||
Cyberark(CyberarkOperationContext),
|
||||
}
|
||||
|
||||
impl SecretOperationContext {
|
||||
pub fn timeout(&self) -> Option<Duration> {
|
||||
match self {
|
||||
Self::Default => None,
|
||||
Self::Aws(context) => context.timeout,
|
||||
Self::Hashicorp(context) => context.timeout,
|
||||
Self::Cyberark(context) => context.timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub struct AwsOperationContext {
|
||||
pub timeout: Option<Duration>,
|
||||
pub region_name: Option<String>,
|
||||
pub role_name: Option<String>,
|
||||
pub session_name: Option<String>,
|
||||
pub external_id: Option<SecretValue>,
|
||||
pub profile_name: Option<String>,
|
||||
pub web_identity_token: Option<SecretValue>,
|
||||
pub sts_endpoint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub struct HashicorpOperationContext {
|
||||
pub timeout: Option<Duration>,
|
||||
pub mount: Option<String>,
|
||||
pub path_prefix: Option<String>,
|
||||
pub data_key: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub struct CyberarkOperationContext {
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Eq, PartialEq)]
|
||||
pub struct SecretWriteContext {
|
||||
pub description: Option<String>,
|
||||
pub tags: BTreeMap<String, String>,
|
||||
pub operation: SecretOperationContext,
|
||||
}
|
||||
|
||||
impl SecretWriteContext {
|
||||
pub fn rotated_from(current_name: &str, operation: SecretOperationContext) -> Self {
|
||||
Self {
|
||||
description: Some(format!("Rotated from {current_name}")),
|
||||
tags: BTreeMap::new(),
|
||||
operation,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -2,11 +2,16 @@
|
|||
|
||||
mod base_secret_manager;
|
||||
mod config;
|
||||
mod context;
|
||||
mod error;
|
||||
mod value;
|
||||
|
||||
pub use base_secret_manager::{BaseSecretManager, async_rotate_secret, validate_secret_name};
|
||||
pub use config::{AccessMode, KeyManagementSettings, KeyManagementSystem};
|
||||
pub use context::{
|
||||
AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext,
|
||||
SecretOperationContext, SecretWriteContext,
|
||||
};
|
||||
pub use error::Error;
|
||||
pub use litellm_auth_types::SecretValue;
|
||||
pub use value::Secret;
|
||||
|
|
|
|||
|
|
@ -1,15 +1,20 @@
|
|||
use litellm_secrets_types::{
|
||||
AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn config_preserves_defaults_nulls_and_serialized_names() {
|
||||
let empty: KeyManagementSettings = serde_json::from_value(json!({})).unwrap();
|
||||
assert_eq!(empty, KeyManagementSettings::default());
|
||||
assert_eq!(empty.access_mode, AccessMode::ReadOnly);
|
||||
assert_eq!(empty.store_virtual_keys, Some(false));
|
||||
assert_eq!(empty.prefix_for_stored_virtual_keys, "litellm/");
|
||||
#[fixture]
|
||||
fn default_settings() -> KeyManagementSettings {
|
||||
serde_json::from_value(json!({})).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn config_preserves_defaults_nulls_and_serialized_names(default_settings: KeyManagementSettings) {
|
||||
assert_eq!(default_settings, KeyManagementSettings::default());
|
||||
assert_eq!(default_settings.access_mode, AccessMode::ReadOnly);
|
||||
assert_eq!(default_settings.store_virtual_keys, Some(false));
|
||||
assert_eq!(default_settings.prefix_for_stored_virtual_keys, "litellm/");
|
||||
let configured: KeyManagementSettings = serde_json::from_value(json!({
|
||||
"hosted_keys": [], "store_virtual_keys": null, "access_mode": "write_only",
|
||||
"aws_web_identity_token": "private-token", "aws_external_id": "private-id",
|
||||
|
|
@ -29,7 +34,7 @@ fn config_preserves_defaults_nulls_and_serialized_names() {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::aws_kms("aws_kms", KeyManagementSystem::AwsKms)]
|
||||
#[case::aws_secret_manager("aws_secret_manager", KeyManagementSystem::AwsSecretManager)]
|
||||
#[case::google_kms("google_kms", KeyManagementSystem::GoogleKms)]
|
||||
|
|
@ -50,11 +55,32 @@ fn key_management_system_serialization_round_trips(
|
|||
assert_eq!(serde_json::to_value(system).unwrap(), name);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn secret_debug_never_exposes_values() {
|
||||
assert!(
|
||||
!format!("{:?}", Secret::String(SecretValue::new("sensitive-value")))
|
||||
.contains("sensitive-value")
|
||||
);
|
||||
assert!(!format!("{:?}", Secret::Bool(true)).contains("true"));
|
||||
#[rstest]
|
||||
#[case::read_only(AccessMode::ReadOnly, true)]
|
||||
#[case::write_only(AccessMode::WriteOnly, false)]
|
||||
#[case::read_and_write(AccessMode::ReadAndWrite, true)]
|
||||
fn access_mode_reports_readability(#[case] mode: AccessMode, #[case] expected: bool) {
|
||||
assert_eq!(mode.readable(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::string(json!("value"), Secret::String(SecretValue::new("value")))]
|
||||
#[case::boolean(json!(true), Secret::Bool(true))]
|
||||
#[case::number(json!(7), Secret::Json(json!(7)))]
|
||||
#[case::null(json!(null), Secret::Json(json!(null)))]
|
||||
#[case::array(json!(["value"]), Secret::Json(json!(["value"])))]
|
||||
#[case::object(json!({"key": "value"}), Secret::Json(json!({"key": "value"})))]
|
||||
fn secret_conversion_preserves_json_types(
|
||||
#[case] value: serde_json::Value,
|
||||
#[case] expected: Secret,
|
||||
) {
|
||||
assert_eq!(Secret::from_json(value), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::string(Secret::String(SecretValue::new("sensitive-value")), "sensitive-value")]
|
||||
#[case::boolean(Secret::Bool(true), "true")]
|
||||
#[case::json(Secret::Json(json!({"private": "value"})), "private")]
|
||||
fn secret_debug_never_exposes_values(#[case] secret: Secret, #[case] sensitive: &str) {
|
||||
assert!(!format!("{secret:?}").contains(sensitive));
|
||||
}
|
||||
|
|
|
|||
110
litellm-rust/crates/secrets-types/tests/context.rs
Normal file
110
litellm-rust/crates/secrets-types/tests/context.rs
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
use std::{collections::BTreeMap, time::Duration};
|
||||
|
||||
use litellm_secrets_types::{
|
||||
AwsOperationContext, CyberarkOperationContext, HashicorpOperationContext,
|
||||
SecretOperationContext, SecretValue, SecretWriteContext,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
#[fixture]
|
||||
fn timeout() -> Duration {
|
||||
Duration::from_secs(30)
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn aws_context(timeout: Duration) -> SecretOperationContext {
|
||||
SecretOperationContext::Aws(AwsOperationContext {
|
||||
timeout: Some(timeout),
|
||||
region_name: Some("us-west-2".into()),
|
||||
role_name: Some("role".into()),
|
||||
external_id: Some(SecretValue::new("external-id")),
|
||||
web_identity_token: Some(SecretValue::new("web-identity-token")),
|
||||
..AwsOperationContext::default()
|
||||
})
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn operation_context_preserves_backend_specific_values_and_redacts_secrets(
|
||||
aws_context: SecretOperationContext,
|
||||
timeout: Duration,
|
||||
) {
|
||||
assert_eq!(aws_context.timeout(), Some(timeout));
|
||||
assert_eq!(
|
||||
aws_context,
|
||||
SecretOperationContext::Aws(AwsOperationContext {
|
||||
timeout: Some(timeout),
|
||||
region_name: Some("us-west-2".into()),
|
||||
role_name: Some("role".into()),
|
||||
external_id: Some(SecretValue::new("external-id")),
|
||||
web_identity_token: Some(SecretValue::new("web-identity-token")),
|
||||
..AwsOperationContext::default()
|
||||
})
|
||||
);
|
||||
let debug = format!("{aws_context:?}");
|
||||
assert!(!debug.contains("external-id"));
|
||||
assert!(!debug.contains("web-identity-token"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default(SecretOperationContext::Default, None)]
|
||||
#[case::aws(
|
||||
SecretOperationContext::Aws(AwsOperationContext {
|
||||
timeout: Some(Duration::from_secs(1)),
|
||||
..AwsOperationContext::default()
|
||||
}),
|
||||
Some(Duration::from_secs(1))
|
||||
)]
|
||||
#[case::hashicorp(
|
||||
SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
timeout: Some(Duration::from_secs(2)),
|
||||
..HashicorpOperationContext::default()
|
||||
}),
|
||||
Some(Duration::from_secs(2))
|
||||
)]
|
||||
#[case::cyberark(
|
||||
SecretOperationContext::Cyberark(CyberarkOperationContext {
|
||||
timeout: Some(Duration::from_secs(3)),
|
||||
}),
|
||||
Some(Duration::from_secs(3))
|
||||
)]
|
||||
fn operation_context_reports_each_backend_timeout(
|
||||
#[case] context: SecretOperationContext,
|
||||
#[case] expected: Option<Duration>,
|
||||
) {
|
||||
assert_eq!(context.timeout(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn write_context_keeps_tags_separate_from_the_operation_context() {
|
||||
let context = SecretWriteContext {
|
||||
description: Some("Managed virtual key".into()),
|
||||
tags: BTreeMap::from([("team".into(), "team-id".into())]),
|
||||
operation: SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
mount: Some("secret".into()),
|
||||
path_prefix: Some("teams".into()),
|
||||
data_key: Some("api_key".into()),
|
||||
..HashicorpOperationContext::default()
|
||||
}),
|
||||
};
|
||||
|
||||
assert_eq!(context.tags.get("team"), Some(&"team-id".into()));
|
||||
assert_eq!(context.operation.timeout(), None);
|
||||
assert_eq!(
|
||||
context.operation,
|
||||
SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
mount: Some("secret".into()),
|
||||
path_prefix: Some("teams".into()),
|
||||
data_key: Some("api_key".into()),
|
||||
..HashicorpOperationContext::default()
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn rotation_write_context_preserves_the_operation_context(aws_context: SecretOperationContext) {
|
||||
let context = SecretWriteContext::rotated_from("current", aws_context.clone());
|
||||
|
||||
assert_eq!(context.description.as_deref(), Some("Rotated from current"));
|
||||
assert!(context.tags.is_empty());
|
||||
assert_eq!(context.operation, aws_context);
|
||||
}
|
||||
|
|
@ -1,12 +1,15 @@
|
|||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use litellm_secrets_types::{
|
||||
BaseSecretManager, Error, SecretValue, async_rotate_secret, validate_secret_name,
|
||||
BaseSecretManager, Error, HashicorpOperationContext, SecretOperationContext, SecretValue,
|
||||
SecretWriteContext, async_rotate_secret, validate_secret_name,
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
struct Manager {
|
||||
step: AtomicUsize,
|
||||
absent_at: Option<usize>,
|
||||
operation: SecretOperationContext,
|
||||
}
|
||||
|
||||
impl BaseSecretManager for Manager {
|
||||
|
|
@ -14,7 +17,12 @@ impl BaseSecretManager for Manager {
|
|||
type WriteResponse = &'static str;
|
||||
type DeleteResponse = ();
|
||||
|
||||
async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
|
||||
async fn async_read_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<Option<SecretValue>, Error> {
|
||||
assert_eq!(context, &self.operation);
|
||||
let step = self.step.fetch_add(1, Ordering::SeqCst);
|
||||
assert_eq!(name, if step == 0 { "old" } else { "new" });
|
||||
Ok((self.absent_at != Some(step)).then(|| SecretValue::new("value")))
|
||||
|
|
@ -24,35 +32,54 @@ impl BaseSecretManager for Manager {
|
|||
&self,
|
||||
name: &str,
|
||||
value: &SecretValue,
|
||||
description: Option<&str>,
|
||||
context: &SecretWriteContext,
|
||||
) -> Result<Self::WriteResponse, Error> {
|
||||
assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 1);
|
||||
assert_eq!(name, "new");
|
||||
assert_eq!(value.expose(), "replacement");
|
||||
assert_eq!(description, Some("Rotated from old"));
|
||||
assert_eq!(context.description.as_deref(), Some("Rotated from old"));
|
||||
assert!(context.tags.is_empty());
|
||||
assert_eq!(context.operation, self.operation);
|
||||
Ok("provider-response")
|
||||
}
|
||||
|
||||
async fn async_delete_secret(
|
||||
&self,
|
||||
name: &str,
|
||||
recovery_window_in_days: i64,
|
||||
recovery_window_in_days: Option<u32>,
|
||||
context: &SecretOperationContext,
|
||||
) -> Result<(), Error> {
|
||||
assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 3);
|
||||
assert_eq!(name, "old");
|
||||
assert_eq!(recovery_window_in_days, 7);
|
||||
assert_eq!(recovery_window_in_days, Some(7));
|
||||
assert_eq!(context, &self.operation);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn replacement() -> SecretValue {
|
||||
SecretValue::new("replacement")
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default(SecretOperationContext::Default)]
|
||||
#[case::backend_specific(SecretOperationContext::Hashicorp(HashicorpOperationContext {
|
||||
mount: Some("alternate".to_owned()),
|
||||
..HashicorpOperationContext::default()
|
||||
}))]
|
||||
#[tokio::test]
|
||||
async fn rotation_verifies_before_deleting_and_returns_provider_response() {
|
||||
async fn rotation_verifies_before_deleting_and_returns_provider_response(
|
||||
replacement: SecretValue,
|
||||
#[case] operation: SecretOperationContext,
|
||||
) {
|
||||
let manager = Manager {
|
||||
step: AtomicUsize::new(0),
|
||||
absent_at: None,
|
||||
operation: operation.clone(),
|
||||
};
|
||||
assert_eq!(
|
||||
async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement"))
|
||||
async_rotate_secret(&manager, "old", "new", &replacement, &operation,)
|
||||
.await
|
||||
.unwrap(),
|
||||
"provider-response"
|
||||
|
|
@ -60,11 +87,12 @@ async fn rotation_verifies_before_deleting_and_returns_provider_response() {
|
|||
assert_eq!(manager.step.load(Ordering::SeqCst), 4);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::current_secret_missing(0, Error::CurrentSecretMissing, 1)]
|
||||
#[case::new_secret_missing(2, Error::NewSecretMissing, 3)]
|
||||
#[tokio::test]
|
||||
async fn missing_old_or_new_value_stops_rotation_before_deletion(
|
||||
replacement: SecretValue,
|
||||
#[case] absent_at: usize,
|
||||
#[case] expected: Error,
|
||||
#[case] calls: usize,
|
||||
|
|
@ -72,32 +100,57 @@ async fn missing_old_or_new_value_stops_rotation_before_deletion(
|
|||
let manager = Manager {
|
||||
step: AtomicUsize::new(0),
|
||||
absent_at: Some(absent_at),
|
||||
operation: SecretOperationContext::default(),
|
||||
};
|
||||
assert_eq!(
|
||||
async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement"))
|
||||
.await
|
||||
.unwrap_err(),
|
||||
async_rotate_secret(
|
||||
&manager,
|
||||
"old",
|
||||
"new",
|
||||
&replacement,
|
||||
&SecretOperationContext::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(manager.step.load(Ordering::SeqCst), calls);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::parent("..")]
|
||||
#[case::parent_prefix("../x")]
|
||||
#[case::parent_segment("x/../y")]
|
||||
#[case::parent_suffix("x/..")]
|
||||
#[case::line_feed("line\n")]
|
||||
#[case::next_line("\u{85}")]
|
||||
#[case::line_separator("\u{2028}")]
|
||||
#[case::paragraph_separator("\u{2029}")]
|
||||
#[case::parent_prefix("../../../other-app/creds")]
|
||||
#[case::nested_parent_prefix("litellm/../../secret")]
|
||||
#[case::parent_segment("foo/../bar")]
|
||||
#[case::parent_suffix("foo/..")]
|
||||
#[case::single_parent_prefix("../foo")]
|
||||
#[case::line_feed("foo\nbar")]
|
||||
#[case::carriage_return("foo\rbar")]
|
||||
#[case::tab("foo\tbar")]
|
||||
#[case::null("foo\0bar")]
|
||||
#[case::delete("foo\u{7f}bar")]
|
||||
#[case::next_line("foo\u{85}bar")]
|
||||
#[case::line_separator("foo\u{2028}bar")]
|
||||
#[case::paragraph_separator("foo\u{2029}bar")]
|
||||
fn names_reject_path_traversal_and_control_characters(#[case] name: &str) {
|
||||
assert_eq!(validate_secret_name(name), Err(Error::UnsafeSecretName));
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[rstest]
|
||||
#[case::plain_alias("plain-alias")]
|
||||
#[case::key_with_digits("my-key-123")]
|
||||
#[case::service_path("prod/my-service-key")]
|
||||
#[case::email_path("team/user@example.com")]
|
||||
#[case::colon("foo: bar")]
|
||||
#[case::spaced_fragment("foo # bar")]
|
||||
#[case::query("foo?evil=1")]
|
||||
#[case::fragment("foo#bar")]
|
||||
#[case::long_alias(&"a".repeat(500))]
|
||||
#[case::embedded_double_dot("release-1.0..2")]
|
||||
#[case::path_separator("folder/key")]
|
||||
#[case::middle_double_dot("my..key")]
|
||||
#[case::leading_double_dot("..foo")]
|
||||
#[case::trailing_double_dot("foo..")]
|
||||
#[case::version_double_dot("v2.0..1-beta")]
|
||||
#[case::empty("")]
|
||||
#[case::three_dots("...")]
|
||||
fn names_allow_safe_values(#[case] name: &str) {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,10 @@ use std::{future::Future, pin::Pin, sync::Arc};
|
|||
|
||||
use litellm_core_utils::settings::Lookup;
|
||||
|
||||
use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue};
|
||||
use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret};
|
||||
|
||||
#[cfg(any(feature = "aws", feature = "google"))]
|
||||
use crate::SecretValue;
|
||||
|
||||
pub trait ExternalSecretManager: Send + Sync {
|
||||
fn system(&self) -> KeyManagementSystem;
|
||||
|
|
@ -17,7 +20,6 @@ pub trait ExternalSecretManager: Send + Sync {
|
|||
|
||||
#[derive(Clone)]
|
||||
pub enum SecretManager {
|
||||
Local,
|
||||
External(Arc<dyn ExternalSecretManager>),
|
||||
#[cfg(feature = "aws")]
|
||||
AwsKms(crate::aws::AwsKms),
|
||||
|
|
@ -38,7 +40,6 @@ pub enum SecretManager {
|
|||
impl SecretManager {
|
||||
pub fn system(&self) -> KeyManagementSystem {
|
||||
match self {
|
||||
Self::Local => KeyManagementSystem::Local,
|
||||
Self::External(manager) => manager.system(),
|
||||
#[cfg(feature = "aws")]
|
||||
Self::AwsKms(_) => KeyManagementSystem::AwsKms,
|
||||
|
|
@ -65,10 +66,6 @@ pub async fn get_secret_from_manager(
|
|||
environment: &(dyn Lookup + Send + Sync),
|
||||
) -> Result<Option<Secret>, Error> {
|
||||
match client {
|
||||
SecretManager::Local => Ok(environment
|
||||
.get(secret_name)
|
||||
.map(SecretValue::new)
|
||||
.map(Secret::String)),
|
||||
SecretManager::External(manager) => {
|
||||
manager
|
||||
.read_secret(secret_name, _settings, environment)
|
||||
|
|
@ -117,10 +114,9 @@ pub async fn get_secret_from_manager(
|
|||
.map(|value| value.map(Secret::String))
|
||||
.map_err(Error::from),
|
||||
#[cfg(feature = "azure")]
|
||||
SecretManager::AzureKeyVault(client) => client
|
||||
.get_secret_from_azure_key_vault(secret_name)
|
||||
.await
|
||||
.map_err(Error::from),
|
||||
SecretManager::AzureKeyVault(client) => {
|
||||
client.get_secret(secret_name).await.map_err(Error::from)
|
||||
}
|
||||
#[cfg(feature = "cyberark")]
|
||||
SecretManager::Cyberark(client) => client
|
||||
.async_read_secret(secret_name)
|
||||
|
|
|
|||
|
|
@ -1,19 +1,14 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use litellm_secrets::{
|
||||
Error, KeyManagementSettings, OidcResolver, Secret, SecretManager, SecretManagerState,
|
||||
SecretResolver, SecretValue, secret_manager_would_be_consulted,
|
||||
Error, OidcResolver, Secret, SecretManagerState, SecretResolver, SecretValue,
|
||||
secret_manager_would_be_consulted,
|
||||
};
|
||||
|
||||
fn resolver(value: Option<&str>, configured: bool) -> SecretResolver {
|
||||
let state = if configured {
|
||||
SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default())
|
||||
} else {
|
||||
SecretManagerState::default()
|
||||
};
|
||||
fn resolver(value: Option<&str>) -> SecretResolver {
|
||||
let value = value.map(str::to_owned);
|
||||
SecretResolver::new(
|
||||
Arc::new(state),
|
||||
Arc::new(SecretManagerState::default()),
|
||||
Arc::new(move |_: &str| value.clone()),
|
||||
OidcResolver::default(),
|
||||
)
|
||||
|
|
@ -30,9 +25,8 @@ fn resolver(value: Option<&str>, configured: bool) -> SecretResolver {
|
|||
async fn conversion_is_explicit_and_independent_of_manager_configuration(
|
||||
#[case] input: &str,
|
||||
#[case] boolean: Option<bool>,
|
||||
#[values(false, true)] configured: bool,
|
||||
) {
|
||||
let resolver = resolver(Some(input), configured);
|
||||
let resolver = resolver(Some(input));
|
||||
assert_eq!(
|
||||
resolver.get_secret("key", None).await.unwrap(),
|
||||
Some(Secret::String(SecretValue::new(input)))
|
||||
|
|
@ -62,8 +56,8 @@ async fn conversion_is_explicit_and_independent_of_manager_configuration(
|
|||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) {
|
||||
let missing = resolver(None, configured);
|
||||
async fn defaults_apply_only_to_absence() {
|
||||
let missing = resolver(None);
|
||||
assert_eq!(missing.get_secret("key", None).await.unwrap(), None);
|
||||
assert_eq!(
|
||||
missing.get_secret_bool("key", Some(false)).await.unwrap(),
|
||||
|
|
@ -92,7 +86,7 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool)
|
|||
);
|
||||
}
|
||||
assert_eq!(
|
||||
resolver(Some(""), configured)
|
||||
resolver(Some(""))
|
||||
.get_secret_str("key", Some(SecretValue::new("default")))
|
||||
.await
|
||||
.unwrap()
|
||||
|
|
@ -103,12 +97,8 @@ async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool)
|
|||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prefix_is_removed_once_and_local_manager_is_not_consulted() {
|
||||
let state = SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default());
|
||||
assert_eq!(
|
||||
state.system(),
|
||||
Some(litellm_secrets::KeyManagementSystem::Local)
|
||||
);
|
||||
async fn prefix_is_removed_once_and_resolved_from_environment() {
|
||||
let state = SecretManagerState::default();
|
||||
assert!(!secret_manager_would_be_consulted(
|
||||
&state,
|
||||
"os.environ/os.environ/KEY"
|
||||
|
|
@ -131,7 +121,7 @@ async fn prefix_is_removed_once_and_local_manager_is_not_consulted() {
|
|||
|
||||
#[tokio::test]
|
||||
async fn resolver_future_can_run_on_a_tokio_worker() {
|
||||
let resolver = resolver(Some("worker-value"), false);
|
||||
let resolver = resolver(Some("worker-value"));
|
||||
let result = tokio::spawn(async move { resolver.get_secret_str("KEY", None).await })
|
||||
.await
|
||||
.unwrap()
|
||||
|
|
@ -142,7 +132,9 @@ async fn resolver_future_can_run_on_a_tokio_worker() {
|
|||
#[cfg(feature = "aws")]
|
||||
mod aws {
|
||||
use super::*;
|
||||
use litellm_secrets::{AccessMode, FailurePolicy, aws::AwsSecretsManagerV2};
|
||||
use litellm_secrets::{
|
||||
AccessMode, FailurePolicy, KeyManagementSettings, SecretManager, aws::AwsSecretsManagerV2,
|
||||
};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
|
||||
|
||||
fn state(server: &MockServer, settings: KeyManagementSettings) -> SecretManagerState {
|
||||
|
|
@ -319,7 +311,9 @@ mod aws {
|
|||
#[case::failure(503)]
|
||||
#[tokio::test]
|
||||
async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) {
|
||||
use litellm_secrets::{FailurePolicy, google::GoogleSecretManager};
|
||||
use litellm_secrets::{
|
||||
FailurePolicy, KeyManagementSettings, SecretManager, google::GoogleSecretManager,
|
||||
};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
|
||||
let server = MockServer::start().await;
|
||||
Mock::given(method("GET"))
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ mod scanner;
|
|||
mod tiktoken;
|
||||
mod unicode_classes;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use byte_level::ByteLevelCounter;
|
||||
use scanner::{SplitPattern, TiktokenCounter};
|
||||
|
||||
|
|
@ -15,22 +17,29 @@ pub use error::Error;
|
|||
|
||||
enum Encoder {
|
||||
HuggingFace {
|
||||
tokenizer: Box<tokenizers::Tokenizer>,
|
||||
tokenizer: Arc<tokenizers::Tokenizer>,
|
||||
byte_level: Option<ByteLevelCounter>,
|
||||
},
|
||||
Tiktoken(TiktokenCounter),
|
||||
}
|
||||
|
||||
/// A count-only tokenizer. Its model tables are immutable, so one built from an already
|
||||
/// loaded model (`from_shared`, `from_*_pairs`) adds only the count-specific tables.
|
||||
pub struct FastTokenizer(Encoder);
|
||||
|
||||
impl FastTokenizer {
|
||||
pub fn from_json(json: &str) -> Result<Self, Error> {
|
||||
let tokenizer = json.parse::<tokenizers::Tokenizer>().map_err(Error::Load)?;
|
||||
Ok(Self::from_shared(Arc::new(tokenizer)))
|
||||
}
|
||||
|
||||
/// Counts with a Hugging Face model another codec already holds; nothing is re-parsed.
|
||||
pub fn from_shared(tokenizer: Arc<tokenizers::Tokenizer>) -> Self {
|
||||
let byte_level = ByteLevelCounter::detect(&tokenizer);
|
||||
Ok(Self(Encoder::HuggingFace {
|
||||
tokenizer: Box::new(tokenizer),
|
||||
Self(Encoder::HuggingFace {
|
||||
tokenizer,
|
||||
byte_level,
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn from_cl100k_ranks(ranks: &str) -> Result<Self, Error> {
|
||||
|
|
@ -41,12 +50,36 @@ impl FastTokenizer {
|
|||
Self::from_ranks(SplitPattern::O200k, ranks)
|
||||
}
|
||||
|
||||
/// `cl100k_base` from ranks another loader already parsed.
|
||||
pub fn from_cl100k_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_pairs(SplitPattern::Cl100k, pairs)
|
||||
}
|
||||
|
||||
/// `o200k_base` (and `o200k_harmony`, whose ordinary tokens are the same) from ranks
|
||||
/// another loader already parsed.
|
||||
pub fn from_o200k_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_pairs(SplitPattern::O200k, pairs)
|
||||
}
|
||||
|
||||
fn from_ranks(split: SplitPattern, ranks: &str) -> Result<Self, Error> {
|
||||
TiktokenCounter::from_ranks(split, ranks)
|
||||
.map(Encoder::Tiktoken)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
fn from_pairs<'a>(
|
||||
split: SplitPattern,
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
TiktokenCounter::from_pairs(split, pairs)
|
||||
.map(Encoder::Tiktoken)
|
||||
.map(Self)
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
match &self.0 {
|
||||
Encoder::Tiktoken(counter) => Ok(counter.count(text)),
|
||||
|
|
|
|||
|
|
@ -39,8 +39,19 @@ pub(super) struct TiktokenCounter {
|
|||
|
||||
impl TiktokenCounter {
|
||||
pub(super) fn from_ranks(split: SplitPattern, rank_file: &str) -> Result<Self, Error> {
|
||||
Self::new(split, MergeRanks::parse(rank_file)?)
|
||||
}
|
||||
|
||||
pub(super) fn from_pairs<'a>(
|
||||
split: SplitPattern,
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], u32)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::new(split, MergeRanks::from_pairs(pairs)?)
|
||||
}
|
||||
|
||||
fn new(split: SplitPattern, ranks: MergeRanks) -> Result<Self, Error> {
|
||||
Ok(Self {
|
||||
ranks: MergeRanks::parse(rank_file)?,
|
||||
ranks,
|
||||
piece_len: split.piece_len(),
|
||||
unicode_classes: UnicodeClasses::get().ok_or(Error::UnicodeClasses)?,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -22,11 +22,25 @@ pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
|
|||
|
||||
impl MergeRanks {
|
||||
pub(super) fn parse(text: &str) -> Result<Self, Error> {
|
||||
let ranks = text
|
||||
.lines()
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(parse_line)
|
||||
.collect::<Result<FxHashMap<_, _>, _>>()?;
|
||||
Self::from_entries(text.lines().filter(|line| !line.is_empty()).map(parse_line))
|
||||
}
|
||||
|
||||
/// The same table from ranks another loader already parsed.
|
||||
pub(super) fn from_pairs<'a>(
|
||||
pairs: impl IntoIterator<Item = (&'a [u8], Rank)>,
|
||||
) -> Result<Self, Error> {
|
||||
Self::from_entries(pairs.into_iter().map(|(bytes, rank)| {
|
||||
if rank == NO_RANK {
|
||||
return Err(Error::Ranks(format!("rank {rank} is reserved")));
|
||||
}
|
||||
Ok((Box::from(bytes), rank))
|
||||
}))
|
||||
}
|
||||
|
||||
fn from_entries(
|
||||
entries: impl Iterator<Item = Result<(Box<[u8]>, Rank), Error>>,
|
||||
) -> Result<Self, Error> {
|
||||
let ranks = entries.collect::<Result<FxHashMap<_, _>, _>>()?;
|
||||
if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
|
||||
return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,5 +6,6 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokenizers.workspace = true
|
||||
|
|
|
|||
|
|
@ -6,4 +6,6 @@ pub enum Error {
|
|||
Load(#[source] tokenizers::Error),
|
||||
#[error("tokenization failed: {0}")]
|
||||
Encode(#[source] tokenizers::Error),
|
||||
#[error("token decoding failed: {0}")]
|
||||
Decode(#[source] tokenizers::Error),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,22 +2,243 @@
|
|||
|
||||
mod error;
|
||||
|
||||
pub use error::Error;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
|
||||
pub use error::Error;
|
||||
use tokenizers::PostProcessor;
|
||||
pub use tokenizers::{
|
||||
AddedToken, EncodeInput, Encoding, InputSequence, PaddingDirection, PaddingParams,
|
||||
PaddingStrategy, TruncationDirection, TruncationParams,
|
||||
};
|
||||
|
||||
pub fn encoding_from_json(json: &str) -> Result<Encoding, Error> {
|
||||
serde_json::from_str(json).map_err(|error| Error::Load(error.into()))
|
||||
}
|
||||
|
||||
pub fn encoding_to_json(encoding: &Encoding) -> Result<String, Error> {
|
||||
serde_json::to_string(encoding).map_err(|error| Error::Load(error.into()))
|
||||
}
|
||||
|
||||
pub struct HuggingFaceTokenizer {
|
||||
tokenizer: Arc<tokenizers::Tokenizer>,
|
||||
special_token_ids: HashSet<u32>,
|
||||
}
|
||||
|
||||
impl HuggingFaceTokenizer {
|
||||
pub fn from_json(json: &str) -> Result<Self, Error> {
|
||||
json.parse::<tokenizers::Tokenizer>()
|
||||
.map(Box::new)
|
||||
.map(Self)
|
||||
.map(Self::new)
|
||||
.map_err(Error::Load)
|
||||
}
|
||||
|
||||
fn new(tokenizer: tokenizers::Tokenizer) -> Self {
|
||||
let special_token_ids: HashSet<u32> = tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.into_iter()
|
||||
.filter_map(|(id, token)| token.special.then_some(id))
|
||||
.collect();
|
||||
Self {
|
||||
tokenizer: Arc::new(tokenizer),
|
||||
special_token_ids,
|
||||
}
|
||||
}
|
||||
|
||||
/// The parsed model, for a count-only counter to share instead of parsing it again.
|
||||
pub fn shared(&self) -> Arc<tokenizers::Tokenizer> {
|
||||
Arc::clone(&self.tokenizer)
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
|
||||
self.0
|
||||
self.tokenizer
|
||||
.encode_fast(text, true)
|
||||
.map(|encoding| encoding.len())
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
self.tokenizer
|
||||
.encode_fast(text, true)
|
||||
.map(|encoding| encoding.get_ids().to_vec())
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode_result<'a>(
|
||||
&self,
|
||||
input: EncodeInput<'a>,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> Result<Encoding, Error> {
|
||||
if fast {
|
||||
return self
|
||||
.tokenizer
|
||||
.encode_fast(input, add_special_tokens)
|
||||
.map_err(Error::Encode);
|
||||
}
|
||||
self.tokenizer
|
||||
.encode_char_offsets(input, add_special_tokens)
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn encode_batch_result<'a>(
|
||||
&self,
|
||||
inputs: Vec<EncodeInput<'a>>,
|
||||
add_special_tokens: bool,
|
||||
fast: bool,
|
||||
) -> Result<Vec<Encoding>, Error> {
|
||||
if fast {
|
||||
return self
|
||||
.tokenizer
|
||||
.encode_batch_fast(inputs, add_special_tokens)
|
||||
.map_err(Error::Encode);
|
||||
}
|
||||
self.tokenizer
|
||||
.encode_batch_char_offsets(inputs, add_special_tokens)
|
||||
.map_err(Error::Encode)
|
||||
}
|
||||
|
||||
pub fn to_json(&self, pretty: bool) -> Result<String, Error> {
|
||||
self.tokenizer.to_string(pretty).map_err(Error::Load)
|
||||
}
|
||||
|
||||
pub fn token_to_id(&self, token: &str) -> Option<u32> {
|
||||
self.tokenizer.token_to_id(token)
|
||||
}
|
||||
|
||||
pub fn id_to_token(&self, id: u32) -> Option<String> {
|
||||
self.tokenizer.id_to_token(id)
|
||||
}
|
||||
|
||||
pub fn vocab(&self, with_added_tokens: bool) -> HashMap<String, u32> {
|
||||
self.tokenizer.get_vocab(with_added_tokens)
|
||||
}
|
||||
|
||||
pub fn vocab_size(&self, with_added_tokens: bool) -> usize {
|
||||
self.tokenizer.get_vocab_size(with_added_tokens)
|
||||
}
|
||||
|
||||
/// The added tokens by id, in id order.
|
||||
pub fn added_tokens_decoder(&self) -> Vec<(u32, AddedToken)> {
|
||||
let mut added: Vec<(u32, AddedToken)> = self
|
||||
.tokenizer
|
||||
.get_added_tokens_decoder()
|
||||
.into_iter()
|
||||
.collect();
|
||||
added.sort_unstable_by_key(|(id, _)| *id);
|
||||
added
|
||||
}
|
||||
|
||||
pub fn padding(&self) -> Option<&PaddingParams> {
|
||||
self.tokenizer.get_padding()
|
||||
}
|
||||
|
||||
pub fn truncation(&self) -> Option<&TruncationParams> {
|
||||
self.tokenizer.get_truncation()
|
||||
}
|
||||
|
||||
/// How many special tokens the post-processor adds to a single sequence or a pair.
|
||||
pub fn num_special_tokens_to_add(&self, is_pair: bool) -> usize {
|
||||
self.tokenizer
|
||||
.get_post_processor()
|
||||
.map_or(0, |processor| processor.added_tokens(is_pair))
|
||||
}
|
||||
|
||||
pub fn encode_special_tokens(&self) -> bool {
|
||||
self.tokenizer.get_encode_special_tokens()
|
||||
}
|
||||
|
||||
pub fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
|
||||
if !skip_special_tokens {
|
||||
return self.tokenizer.decode(ids, false).map_err(Error::Decode);
|
||||
}
|
||||
let filtered_ids: Vec<u32> = ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|id| !self.special_token_ids.contains(id))
|
||||
.collect();
|
||||
self.tokenizer
|
||||
.decode(&filtered_ids, true)
|
||||
.map_err(Error::Decode)
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str {
|
||||
"huggingface"
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn codecs_round_trip_and_skip_special_tokens() {
|
||||
let json = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
|
||||
));
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
let ids = tokenizer.encode("<SOS>hello<EOT>").unwrap();
|
||||
|
||||
assert!(tokenizer.decode(&ids, false).unwrap().contains("<SOS>"));
|
||||
assert_eq!(tokenizer.decode(&ids, true).unwrap(), "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_filters_special_added_tokens() {
|
||||
let json = r#"{
|
||||
"version": "1.0",
|
||||
"truncation": null,
|
||||
"padding": null,
|
||||
"added_tokens": [
|
||||
{
|
||||
"id": 1,
|
||||
"content": "<s>",
|
||||
"single_word": false,
|
||||
"lstrip": false,
|
||||
"rstrip": false,
|
||||
"normalized": false,
|
||||
"special": true
|
||||
}
|
||||
],
|
||||
"normalizer": null,
|
||||
"pre_tokenizer": {"type": "Whitespace"},
|
||||
"post_processor": null,
|
||||
"decoder": null,
|
||||
"model": {
|
||||
"type": "WordLevel",
|
||||
"vocab": {"<unk>": 0, "<s>": 1, "hello": 2},
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
}"#;
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
|
||||
assert!(!tokenizer.decode(&[1, 2], true).unwrap().contains("<s>"));
|
||||
assert!(tokenizer.decode(&[1, 2], false).unwrap().contains("<s>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vocabulary_lookups_mirror_the_tokenizers_api() {
|
||||
let json = include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
|
||||
));
|
||||
let tokenizer = HuggingFaceTokenizer::from_json(json).unwrap();
|
||||
let ids = tokenizer.encode("hello").unwrap();
|
||||
|
||||
let token = tokenizer.id_to_token(ids[0]).unwrap();
|
||||
assert_eq!(tokenizer.token_to_id(&token), Some(ids[0]));
|
||||
assert_eq!(tokenizer.id_to_token(u32::MAX), None);
|
||||
assert_eq!(tokenizer.vocab(true).len(), tokenizer.vocab_size(true));
|
||||
assert!(tokenizer.vocab_size(true) >= tokenizer.vocab_size(false));
|
||||
let added = tokenizer.added_tokens_decoder();
|
||||
assert!(added.windows(2).all(|pair| pair[0].0 < pair[1].0));
|
||||
assert!(added.iter().any(|(_, token)| token.special));
|
||||
assert!(tokenizer.padding().is_none());
|
||||
assert!(tokenizer.truncation().is_none());
|
||||
assert!(!tokenizer.encode_special_tokens());
|
||||
assert_eq!(
|
||||
tokenizer.num_special_tokens_to_add(false),
|
||||
tokenizer.encode("").unwrap().len()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,5 +6,8 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64.workspace = true
|
||||
once_cell = "1.21.3"
|
||||
rustc-hash = "2.1.3"
|
||||
thiserror.workspace = true
|
||||
tiktoken-rs.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,27 +1,125 @@
|
|||
#![forbid(unsafe_code)]
|
||||
|
||||
mod error;
|
||||
mod ranks;
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
pub use error::UnsupportedTokenizer;
|
||||
pub use ranks::{LoadError, Vocabulary};
|
||||
|
||||
pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE);
|
||||
pub struct TiktokenTokenizer {
|
||||
encoder: &'static tiktoken_rs::CoreBPE,
|
||||
/// Present for encodings built from a rank file; the embedded tiktoken-rs singletons
|
||||
/// behind [`from_name`](Self::from_name) keep their ranks private.
|
||||
vocabulary: Option<&'static Vocabulary>,
|
||||
name: &'static str,
|
||||
}
|
||||
|
||||
impl TiktokenTokenizer {
|
||||
/// Builds `name` from its packaged rank file (read through `load`), once per process.
|
||||
/// The tokenizer reports the requested name, so `gpt2` stays `gpt2` like tiktoken does.
|
||||
pub fn from_cached_ranks(
|
||||
name: &str,
|
||||
load: impl FnOnce(&str) -> std::io::Result<String>,
|
||||
) -> Result<Self, LoadError> {
|
||||
let (loaded, name) = ranks::load(name, load)?;
|
||||
Ok(Self {
|
||||
encoder: &loaded.bpe,
|
||||
vocabulary: Some(&loaded.vocabulary),
|
||||
name,
|
||||
})
|
||||
}
|
||||
|
||||
/// The encodings tiktoken-rs embeds, for hosts without the packaged rank files.
|
||||
pub fn from_name(name: &str) -> Result<Self, UnsupportedTokenizer> {
|
||||
let tokenizer = match name {
|
||||
"cl100k_base" => tiktoken_rs::cl100k_base_singleton(),
|
||||
"o200k_base" => tiktoken_rs::o200k_base_singleton(),
|
||||
"o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(),
|
||||
"p50k_base" => tiktoken_rs::p50k_base_singleton(),
|
||||
"p50k_edit" => tiktoken_rs::p50k_edit_singleton(),
|
||||
"r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(),
|
||||
let (encoder, name) = match name {
|
||||
"cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"),
|
||||
"o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"),
|
||||
"o200k_harmony" => (tiktoken_rs::o200k_harmony_singleton(), "o200k_harmony"),
|
||||
"p50k_base" => (tiktoken_rs::p50k_base_singleton(), "p50k_base"),
|
||||
"p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"),
|
||||
"r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"),
|
||||
"gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"),
|
||||
_ => return Err(UnsupportedTokenizer(name.to_owned())),
|
||||
};
|
||||
Ok(Self(tokenizer))
|
||||
Ok(Self {
|
||||
encoder,
|
||||
vocabulary: None,
|
||||
name,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn vocabulary(&self) -> Option<&Vocabulary> {
|
||||
self.vocabulary
|
||||
}
|
||||
|
||||
pub fn count_tokens(&self, text: &str) -> usize {
|
||||
self.0.count_ordinary(text)
|
||||
self.encoder.count_ordinary(text)
|
||||
}
|
||||
|
||||
pub fn encode(&self, text: &str) -> Vec<u32> {
|
||||
self.encoder.encode_ordinary(text)
|
||||
}
|
||||
|
||||
pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result<Vec<u32>, String> {
|
||||
let allowed = allowed.iter().map(String::as_str).collect();
|
||||
self.encoder
|
||||
.encode(text, &allowed)
|
||||
.map(|(ids, _)| ids)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn special_tokens(&self) -> HashSet<String> {
|
||||
self.encoder
|
||||
.special_tokens()
|
||||
.into_iter()
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// tiktoken's `encode_with_unstable`: the stable prefix of `text`'s tokens and every
|
||||
/// token sequence the unstable tail could still become, sorted for a stable order.
|
||||
pub fn encode_with_unstable(
|
||||
&self,
|
||||
text: &str,
|
||||
allowed: &[String],
|
||||
) -> (Vec<u32>, Vec<Vec<u32>>) {
|
||||
let allowed = allowed.iter().map(String::as_str).collect();
|
||||
let (stable, completions) = self.encoder._encode_unstable_native(text, &allowed);
|
||||
let mut completions: Vec<Vec<u32>> = completions.into_iter().collect();
|
||||
completions.sort_unstable();
|
||||
(stable, completions)
|
||||
}
|
||||
|
||||
pub fn decode_bytes(&self, ids: &[u32]) -> Result<Vec<u8>, String> {
|
||||
self.encoder
|
||||
.decode_bytes(ids)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn decode(&self, ids: &[u32]) -> Result<String, String> {
|
||||
self.encoder
|
||||
.decode_bytes(ids)
|
||||
.map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &str {
|
||||
self.name
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encoding_for_model(model: &str) -> Option<&'static str> {
|
||||
match tiktoken_rs::tokenizer::get_tokenizer(model)? {
|
||||
tiktoken_rs::tokenizer::Tokenizer::Cl100kBase => Some("cl100k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::O200kBase => Some("o200k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::O200kHarmony => Some("o200k_harmony"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::P50kBase => Some("p50k_base"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::P50kEdit => Some("p50k_edit"),
|
||||
tiktoken_rs::tokenizer::Tokenizer::R50kBase | tiktoken_rs::tokenizer::Tokenizer::Gpt2 => {
|
||||
Some("r50k_base")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -66,5 +164,75 @@ mod tests {
|
|||
panic!("unknown encoding must be rejected");
|
||||
};
|
||||
assert_eq!(name, "unknown-encoding");
|
||||
assert_eq!(TiktokenTokenizer::from_name("gpt2").unwrap().name(), "gpt2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codecs_round_trip_named_encodings() {
|
||||
let encodings = [
|
||||
"cl100k_base",
|
||||
"o200k_base",
|
||||
"o200k_harmony",
|
||||
"p50k_base",
|
||||
"p50k_edit",
|
||||
"r50k_base",
|
||||
"gpt2",
|
||||
];
|
||||
let texts = ["hello world", "café 漢字 مرحبا 🙂", "line one\nline two"];
|
||||
for name in encodings {
|
||||
let tokenizer = TiktokenTokenizer::from_name(name).unwrap();
|
||||
for text in texts {
|
||||
assert_eq!(
|
||||
tokenizer.decode(&tokenizer.encode(text)).unwrap(),
|
||||
text,
|
||||
"{name}: {text:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoding_token_prefixes_replaces_incomplete_utf8() {
|
||||
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
|
||||
let reference = tiktoken_rs::cl100k_base_singleton();
|
||||
let ids = tokenizer.encode("🙂漢字");
|
||||
for end in 1..=ids.len() {
|
||||
let bytes = reference.decode_bytes(&ids[..end]).unwrap();
|
||||
assert_eq!(
|
||||
tokenizer.decode(&ids[..end]).unwrap(),
|
||||
String::from_utf8_lossy(&bytes),
|
||||
);
|
||||
}
|
||||
assert!(tokenizer.decode(&[u32::MAX]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unstable_encoding_prefixes_stay_consistent_with_full_encoding() {
|
||||
let tokenizer = TiktokenTokenizer::from_name("cl100k_base").unwrap();
|
||||
let text = "hello fanta";
|
||||
let (stable, completions) = tokenizer.encode_with_unstable(text, &[]);
|
||||
assert!(
|
||||
text.as_bytes()
|
||||
.starts_with(&tokenizer.decode_bytes(&stable).unwrap())
|
||||
);
|
||||
assert!(!completions.is_empty());
|
||||
for completion in &completions {
|
||||
let mut ids = stable.clone();
|
||||
ids.extend(completion);
|
||||
assert!(
|
||||
tokenizer
|
||||
.decode_bytes(&ids)
|
||||
.unwrap()
|
||||
.starts_with(text.as_bytes())
|
||||
);
|
||||
}
|
||||
assert!(completions.windows(2).all(|pair| pair[0] < pair[1]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoding_for_model_maps_known_models() {
|
||||
assert_eq!(encoding_for_model("gpt-4o"), Some("o200k_base"));
|
||||
assert_eq!(encoding_for_model("text-davinci-003"), Some("p50k_base"));
|
||||
assert_eq!(encoding_for_model("unknown-model"), None);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
340
litellm-rust/crates/token-counter-tiktoken/src/ranks.rs
Normal file
340
litellm-rust/crates/token-counter-tiktoken/src/ranks.rs
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
use base64::{Engine, engine::general_purpose::STANDARD};
|
||||
use once_cell::sync::OnceCell;
|
||||
use rustc_hash::FxHashMap;
|
||||
use thiserror::Error;
|
||||
use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank};
|
||||
|
||||
use crate::UnsupportedTokenizer;
|
||||
|
||||
const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4";
|
||||
const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790";
|
||||
const P50K: &str = "ec7223a39ce59f226a68acc30dc1af2788490e15";
|
||||
const LEGACY_PATTERN: &str =
|
||||
r"'(?:[sdmt]|ll|ve|re)| ?\p{L}++| ?\p{N}++| ?[^\s\p{L}\p{N}]++|\s++$|\s+(?!\S)|\s";
|
||||
const CL100K_PATTERN: &str = r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?+\p{L}++|\p{N}{1,3}+| ?[^\s\p{L}\p{N}]++[\r\n]*+|\s++$|\s*[\r\n]|\s+(?!\S)|\s";
|
||||
|
||||
static CL100K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static O200K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static HARMONY_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static P50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static EDIT_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
static R50K_ENCODER: OnceCell<Loaded> = OnceCell::new();
|
||||
|
||||
/// One encoding built from a rank file: the BPE engine plus the vocabulary it was built
|
||||
/// from, kept because `CoreBPE` does not expose its ranks and tiktoken's Python API does
|
||||
/// (`token_byte_values`, `encode_single_token`, `max_token_value`, `_special_tokens`).
|
||||
pub(super) struct Loaded {
|
||||
pub(super) bpe: CoreBPE,
|
||||
pub(super) vocabulary: Vocabulary,
|
||||
}
|
||||
|
||||
/// The byte-level vocabulary of a tiktoken encoding.
|
||||
pub struct Vocabulary {
|
||||
ranks: FxHashMap<Vec<u8>, Rank>,
|
||||
special_tokens: FxHashMap<String, Rank>,
|
||||
max_token_value: Rank,
|
||||
}
|
||||
|
||||
impl Vocabulary {
|
||||
/// Every mergeable token's bytes, sorted bytewise like tiktoken's `token_byte_values`.
|
||||
pub fn token_byte_values(&self) -> Vec<Vec<u8>> {
|
||||
let mut values: Vec<Vec<u8>> = self.ranks.keys().cloned().collect();
|
||||
values.sort_unstable();
|
||||
values
|
||||
}
|
||||
|
||||
/// The rank of one whole token: a mergeable piece first, then a special token's text.
|
||||
pub fn encode_single_token(&self, piece: &[u8]) -> Option<Rank> {
|
||||
if let Some(rank) = self.ranks.get(piece) {
|
||||
return Some(*rank);
|
||||
}
|
||||
std::str::from_utf8(piece)
|
||||
.ok()
|
||||
.and_then(|text| self.special_tokens.get(text).copied())
|
||||
}
|
||||
|
||||
pub fn max_token_value(&self) -> Rank {
|
||||
self.max_token_value
|
||||
}
|
||||
|
||||
/// Every mergeable token with its rank, for building other tables from one parse.
|
||||
pub fn ranks(&self) -> impl Iterator<Item = (&[u8], Rank)> + '_ {
|
||||
self.ranks
|
||||
.iter()
|
||||
.map(|(bytes, rank)| (bytes.as_slice(), *rank))
|
||||
}
|
||||
|
||||
/// The special tokens with their ranks, tiktoken's `_special_tokens`.
|
||||
pub fn special_tokens(&self) -> impl Iterator<Item = (&str, Rank)> + '_ {
|
||||
self.special_tokens
|
||||
.iter()
|
||||
.map(|(token, rank)| (token.as_str(), *rank))
|
||||
}
|
||||
|
||||
pub fn is_special_token(&self, rank: Rank) -> bool {
|
||||
self.special_tokens.values().any(|special| *special == rank)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum LoadError {
|
||||
#[error(transparent)]
|
||||
Unsupported(#[from] UnsupportedTokenizer),
|
||||
#[error("failed to load tiktoken ranks: {0}")]
|
||||
Ranks(String),
|
||||
}
|
||||
|
||||
/// Loads `name` once per process. The returned name is the one requested (`gpt2` stays
|
||||
/// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share
|
||||
/// one cached encoder.
|
||||
pub(super) fn load(
|
||||
name: &str,
|
||||
load_file: impl FnOnce(&str) -> std::io::Result<String>,
|
||||
) -> Result<(&'static Loaded, &'static str), LoadError> {
|
||||
let (requested, canonical, file, cache) = match name {
|
||||
"cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER),
|
||||
"o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER),
|
||||
"o200k_harmony" => ("o200k_harmony", "o200k_harmony", O200K, &HARMONY_ENCODER),
|
||||
"p50k_base" => ("p50k_base", "p50k_base", P50K, &P50K_ENCODER),
|
||||
"p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER),
|
||||
"r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER),
|
||||
"gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER),
|
||||
_ => return Err(UnsupportedTokenizer(name.to_owned()).into()),
|
||||
};
|
||||
let loaded = cache.get_or_try_init(|| {
|
||||
let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
build(canonical, &ranks)
|
||||
})?;
|
||||
Ok((loaded, requested))
|
||||
}
|
||||
|
||||
fn build(name: &str, ranks: &str) -> Result<Loaded, LoadError> {
|
||||
let parsed = ranks
|
||||
.lines()
|
||||
.map(parse_rank)
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let encoder: FxHashMap<_, _> = parsed
|
||||
.into_iter()
|
||||
.filter(|(_, rank)| name != "r50k_base" || *rank < 50256)
|
||||
.collect();
|
||||
if encoder
|
||||
.values()
|
||||
.collect::<std::collections::HashSet<_>>()
|
||||
.len()
|
||||
!= encoder.len()
|
||||
|| (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..]))
|
||||
{
|
||||
return Err(LoadError::Ranks("invalid vocabulary ranks".into()));
|
||||
}
|
||||
let (pattern, specials): (&str, &[(&str, Rank)]) = match name {
|
||||
"cl100k_base" => (
|
||||
CL100K_PATTERN,
|
||||
&[
|
||||
("<|endoftext|>", 100257),
|
||||
("<|fim_prefix|>", 100258),
|
||||
("<|fim_middle|>", 100259),
|
||||
("<|fim_suffix|>", 100260),
|
||||
("<|endofprompt|>", 100276),
|
||||
],
|
||||
),
|
||||
"o200k_base" => (
|
||||
O200K_BASE_PAT_STR,
|
||||
&[("<|endoftext|>", 199999), ("<|endofprompt|>", 200018)],
|
||||
),
|
||||
"o200k_harmony" => (
|
||||
O200K_BASE_PAT_STR,
|
||||
&[
|
||||
("<|startoftext|>", 199998),
|
||||
("<|endoftext|>", 199999),
|
||||
("<|reserved_200000|>", 200000),
|
||||
("<|reserved_200001|>", 200001),
|
||||
("<|return|>", 200002),
|
||||
("<|constrain|>", 200003),
|
||||
("<|reserved_200004|>", 200004),
|
||||
("<|channel|>", 200005),
|
||||
("<|start|>", 200006),
|
||||
("<|end|>", 200007),
|
||||
("<|message|>", 200008),
|
||||
("<|reserved_200009|>", 200009),
|
||||
("<|reserved_200010|>", 200010),
|
||||
("<|reserved_200011|>", 200011),
|
||||
("<|call|>", 200012),
|
||||
],
|
||||
),
|
||||
"p50k_edit" => (
|
||||
LEGACY_PATTERN,
|
||||
&[
|
||||
("<|endoftext|>", 50256),
|
||||
("<|fim_prefix|>", 50281),
|
||||
("<|fim_middle|>", 50282),
|
||||
("<|fim_suffix|>", 50283),
|
||||
],
|
||||
),
|
||||
_ => (LEGACY_PATTERN, &[("<|endoftext|>", 50256)]),
|
||||
};
|
||||
let reserved = (200013..=201087)
|
||||
.filter(|_| name == "o200k_harmony")
|
||||
.map(|rank| (format!("<|reserved_{rank}|>"), rank));
|
||||
let special_tokens: FxHashMap<String, Rank> = specials
|
||||
.iter()
|
||||
.map(|(token, rank)| ((*token).to_owned(), *rank))
|
||||
.chain(reserved)
|
||||
.collect();
|
||||
let max_token_value = encoder
|
||||
.values()
|
||||
.chain(special_tokens.values())
|
||||
.copied()
|
||||
.max()
|
||||
.ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?;
|
||||
let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern)
|
||||
.map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
Ok(Loaded {
|
||||
bpe,
|
||||
vocabulary: Vocabulary {
|
||||
ranks: encoder,
|
||||
special_tokens,
|
||||
max_token_value,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_rank(line: &str) -> Result<(Vec<u8>, Rank), LoadError> {
|
||||
let (token, rank) = line
|
||||
.split_once(' ')
|
||||
.ok_or_else(|| LoadError::Ranks("missing rank".into()))?;
|
||||
let bytes = STANDARD
|
||||
.decode(token)
|
||||
.map_err(|error| LoadError::Ranks(error.to_string()))?;
|
||||
let rank = rank
|
||||
.parse()
|
||||
.map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?;
|
||||
Ok((bytes, rank))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::TiktokenTokenizer;
|
||||
|
||||
fn read_packaged_ranks(file: &str) -> std::io::Result<String> {
|
||||
std::fs::read_to_string(
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../../litellm/litellm_core_utils/tokenizers")
|
||||
.join(file),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn packaged_encodings_match_embedded_encodings_and_reuse_successful_loads() {
|
||||
for name in [
|
||||
"cl100k_base",
|
||||
"o200k_base",
|
||||
"o200k_harmony",
|
||||
"p50k_base",
|
||||
"p50k_edit",
|
||||
"r50k_base",
|
||||
"gpt2",
|
||||
] {
|
||||
if name != "gpt2" {
|
||||
assert!(
|
||||
TiktokenTokenizer::from_cached_ranks(name, |_| {
|
||||
Err(std::io::Error::other("unreadable vocabulary"))
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
let loads = std::sync::atomic::AtomicUsize::new(0);
|
||||
let barrier = std::sync::Barrier::new(4);
|
||||
let encoders = std::thread::scope(|scope| {
|
||||
let tasks: Vec<_> = (0..4)
|
||||
.map(|_| {
|
||||
scope.spawn(|| {
|
||||
barrier.wait();
|
||||
TiktokenTokenizer::from_cached_ranks(name, |file| {
|
||||
loads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
read_packaged_ranks(file)
|
||||
})
|
||||
.unwrap()
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
tasks
|
||||
.into_iter()
|
||||
.map(|task| task.join().unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
assert_eq!(loads.into_inner(), usize::from(name != "gpt2"));
|
||||
let actual = &encoders[0];
|
||||
let expected = TiktokenTokenizer::from_name(name).unwrap();
|
||||
assert_eq!(actual.special_tokens(), expected.special_tokens());
|
||||
let specials: Vec<_> = expected.special_tokens().into_iter().collect();
|
||||
let special_text = specials.join(" ");
|
||||
assert_eq!(
|
||||
actual.encode_special(&special_text, &specials).unwrap(),
|
||||
expected.encode_special(&special_text, &specials).unwrap()
|
||||
);
|
||||
for text in [
|
||||
"",
|
||||
"café 漢字 ع 🙂",
|
||||
"a\r\nb\t ",
|
||||
" hello 123456789",
|
||||
&special_text,
|
||||
] {
|
||||
let ids = expected.encode(text);
|
||||
assert_eq!(actual.encode(text), ids, "{name}: {text:?}");
|
||||
assert_eq!(actual.count_tokens(text), ids.len(), "{name}: {text:?}");
|
||||
assert_eq!(
|
||||
actual.decode_bytes(&ids).unwrap(),
|
||||
expected.decode_bytes(&ids).unwrap()
|
||||
);
|
||||
}
|
||||
let cached = TiktokenTokenizer::from_cached_ranks(name, |_| {
|
||||
panic!("reloaded cached vocabulary")
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(cached.encode("cached"), expected.encode("cached"));
|
||||
assert_eq!(cached.name(), name);
|
||||
assert!(expected.vocabulary().is_none());
|
||||
assert_vocabulary_lookups(name, actual);
|
||||
}
|
||||
}
|
||||
|
||||
/// The token-level lookups tiktoken's Python `Encoding` exposes, checked against the
|
||||
/// encoder itself and against the known vocabulary sizes.
|
||||
fn assert_vocabulary_lookups(name: &str, tokenizer: &TiktokenTokenizer) {
|
||||
let max_token_value = match name {
|
||||
"cl100k_base" => 100_276,
|
||||
"o200k_base" => 200_018,
|
||||
"o200k_harmony" => 201_087,
|
||||
"p50k_base" => 50_280,
|
||||
"p50k_edit" => 50_283,
|
||||
"r50k_base" | "gpt2" => 50_256,
|
||||
_ => unreachable!("{name}"),
|
||||
};
|
||||
let vocabulary = tokenizer.vocabulary().unwrap();
|
||||
assert_eq!(vocabulary.max_token_value(), max_token_value, "{name}");
|
||||
let values = vocabulary.token_byte_values();
|
||||
assert!(values.windows(2).all(|pair| pair[0] < pair[1]), "{name}");
|
||||
for piece in values.iter().step_by(997) {
|
||||
let rank = vocabulary.encode_single_token(piece).unwrap();
|
||||
assert_eq!(tokenizer.decode_bytes(&[rank]).unwrap(), *piece, "{name}");
|
||||
assert!(!vocabulary.is_special_token(rank), "{name}");
|
||||
}
|
||||
for (token, rank) in vocabulary.special_tokens() {
|
||||
assert_eq!(vocabulary.encode_single_token(token.as_bytes()), Some(rank));
|
||||
assert!(vocabulary.is_special_token(rank), "{name}: {token}");
|
||||
}
|
||||
assert_eq!(vocabulary.encode_single_token(b"<|not-a-token|>"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_ranks_return_errors_instead_of_panicking() {
|
||||
for ranks in ["", "IQ==", "IQ== x", "!!! 1", "IQ== 1"] {
|
||||
assert!(build("cl100k_base", ranks).is_err());
|
||||
}
|
||||
let repeated_rank = (0..=u8::MAX)
|
||||
.map(|byte| format!("{} 0\n", STANDARD.encode([byte])))
|
||||
.collect::<String>();
|
||||
assert!(build("cl100k_base", &repeated_rank).is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,10 @@
|
|||
# Token counting
|
||||
|
||||
`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface
|
||||
`Tokenizer` is the text-counting interface. `TextCodec` adds encoding, decoding, and a name. `TokenCounter` applies LiteLLM request, message, and tool accounting using any `Tokenizer`
|
||||
|
||||
Counts follow the codec: tiktoken treats special-token spellings as ordinary text, while Hugging Face applies its added tokens, post-processing, padding, and truncation. `fast=True` preserves those semantics and requests acceleration where available. Unsupported configurations use the normal codec, including tiktoken encodings without a scanner and builds without the `fast` feature. Invalid input and process-guard errors still propagate. Runtime request counting currently uses the normal codec; the custom accelerator is retained for explicit use and testing
|
||||
|
||||
`FastCounter: TextCodec` exposes an optional accelerator over a loaded codec. `None` means callers should use that codec. The Python bridge caches this selection per immutable tokenizer, shares it with request counters, and initializes it with the GIL released. Hugging Face can also choose the full encoder per input when added tokens require it
|
||||
|
||||
The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation
|
||||
|
||||
|
|
@ -8,7 +12,9 @@ The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through t
|
|||
|
||||
The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2`
|
||||
|
||||
All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
|
||||
All three backends are enabled by default in this crate and the Python extension. With `default-features = false`, Rust callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
|
||||
|
||||
Python `tiktoken` and `tokenizers` remain runtime dependencies and the default implementations. The catalog independently selects the tokenizer and request-counting routes. Enabling Rust changes factory dispatch; existing tokenizer objects keep their backend. Native Hugging Face wrappers provide an immutable encoding and decoding API, while training and mutable configuration remain available through the Python backend
|
||||
|
||||
Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ pub enum Error {
|
|||
JsonUtf8(#[source] FromUtf8Error),
|
||||
#[error("tokenization failed: {0}")]
|
||||
Encode(#[source] Box<dyn std::error::Error + Send + Sync>),
|
||||
#[error("token decoding failed: {0}")]
|
||||
Decode(String),
|
||||
#[error("token counting task failed: {0}")]
|
||||
Task(String),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,31 @@
|
|||
use litellm_token_counter_fast::Error as BackendError;
|
||||
pub use litellm_token_counter_fast::FastTokenizer;
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
pub trait FastCounter: TextCodec {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer>;
|
||||
}
|
||||
|
||||
#[cfg(feature = "huggingface")]
|
||||
impl FastCounter for crate::huggingface::HuggingFaceTokenizer {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
Some(FastTokenizer::from_shared(self.shared()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tiktoken")]
|
||||
impl FastCounter for crate::tiktoken::TiktokenTokenizer {
|
||||
fn fast_counter(&self) -> Option<FastTokenizer> {
|
||||
let vocabulary = self.vocabulary()?;
|
||||
match self.name() {
|
||||
"cl100k_base" => FastTokenizer::from_cl100k_pairs(vocabulary.ranks()),
|
||||
"o200k_base" | "o200k_harmony" => FastTokenizer::from_o200k_pairs(vocabulary.ranks()),
|
||||
_ => return None,
|
||||
}
|
||||
.ok()
|
||||
}
|
||||
}
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_json_fast(tokenizer_json: &str) -> Result<Self, Error> {
|
||||
|
|
@ -39,3 +63,65 @@ impl From<BackendError> for Error {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "huggingface", feature = "tiktoken"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::huggingface::HuggingFaceTokenizer;
|
||||
use crate::tiktoken::TiktokenTokenizer;
|
||||
|
||||
const TEXTS: [&str; 4] = [
|
||||
"",
|
||||
"hello world <|endoftext|>",
|
||||
"café 漢字 ع 🙂 line\r\n indented 123456789",
|
||||
"<SOS>system<EOT> a\u{301} fi",
|
||||
];
|
||||
|
||||
fn packaged(file: &str) -> String {
|
||||
std::fs::read_to_string(format!(
|
||||
"{}/../../../litellm/litellm_core_utils/tokenizers/{file}",
|
||||
env!("CARGO_MANIFEST_DIR")
|
||||
))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fast_counters_derived_from_codecs_count_like_the_codecs() {
|
||||
let huggingface =
|
||||
HuggingFaceTokenizer::from_json(&packaged("anthropic_tokenizer.json")).unwrap();
|
||||
let fast = huggingface.fast_counter().unwrap();
|
||||
for text in TEXTS {
|
||||
assert_eq!(
|
||||
fast.count_tokens(text).unwrap(),
|
||||
Tokenizer::count_tokens(&huggingface, text).unwrap(),
|
||||
"{text:?}"
|
||||
);
|
||||
}
|
||||
|
||||
for name in ["cl100k_base", "o200k_base", "o200k_harmony"] {
|
||||
let tiktoken =
|
||||
TiktokenTokenizer::from_cached_ranks(name, |file| Ok(packaged(file))).unwrap();
|
||||
let fast = tiktoken.fast_counter().unwrap();
|
||||
for text in TEXTS {
|
||||
assert_eq!(
|
||||
fast.count_tokens(text).unwrap(),
|
||||
tiktoken.count_tokens(text),
|
||||
"{name}: {text:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encodings_without_a_fast_scanner_keep_the_codec() {
|
||||
let tiktoken =
|
||||
TiktokenTokenizer::from_cached_ranks("p50k_base", |file| Ok(packaged(file))).unwrap();
|
||||
assert!(tiktoken.fast_counter().is_none());
|
||||
assert!(
|
||||
TiktokenTokenizer::from_name("cl100k_base")
|
||||
.unwrap()
|
||||
.fast_counter()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
use litellm_token_counter_huggingface::Error as BackendError;
|
||||
pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
|
||||
pub use litellm_token_counter_huggingface::{
|
||||
AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection,
|
||||
PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json,
|
||||
encoding_to_json,
|
||||
};
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
|
||||
|
|
@ -17,11 +21,26 @@ impl Tokenizer for HuggingFaceTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
impl TextCodec for HuggingFaceTokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
HuggingFaceTokenizer::encode(self, text).map_err(Error::from)
|
||||
}
|
||||
|
||||
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error> {
|
||||
HuggingFaceTokenizer::decode(self, ids, skip_special_tokens).map_err(Error::from)
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
HuggingFaceTokenizer::name(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<BackendError> for Error {
|
||||
fn from(error: BackendError) -> Self {
|
||||
match error {
|
||||
BackendError::Load(source) => Self::Load(source),
|
||||
BackendError::Encode(source) => Self::Encode(source),
|
||||
BackendError::Decode(source) => Self::Decode(source.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,5 +20,5 @@ pub mod tiktoken;
|
|||
|
||||
pub use counter::{InputTokenCount, TokenCounter};
|
||||
pub use error::Error;
|
||||
pub use tokenizer::Tokenizer;
|
||||
pub use tokenizer::{TextCodec, Tokenizer};
|
||||
pub use types::CountableRequest;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
pub use litellm_token_counter_tiktoken::TiktokenTokenizer;
|
||||
use litellm_token_counter_tiktoken::UnsupportedTokenizer;
|
||||
use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer};
|
||||
pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model};
|
||||
|
||||
use crate::{Error, TokenCounter, Tokenizer};
|
||||
use crate::{Error, TextCodec, TokenCounter, Tokenizer};
|
||||
|
||||
impl TokenCounter {
|
||||
pub fn from_tiktoken(encoding: &str) -> Result<Self, Error> {
|
||||
|
|
@ -17,8 +17,31 @@ impl Tokenizer for TiktokenTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
impl TextCodec for TiktokenTokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error> {
|
||||
Ok(TiktokenTokenizer::encode(self, text))
|
||||
}
|
||||
|
||||
fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result<String, Error> {
|
||||
TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string()))
|
||||
}
|
||||
|
||||
fn name(&self) -> &str {
|
||||
TiktokenTokenizer::name(self)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<UnsupportedTokenizer> for Error {
|
||||
fn from(error: UnsupportedTokenizer) -> Self {
|
||||
Self::UnsupportedTokenizer(error.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<LoadError> for Error {
|
||||
fn from(error: LoadError) -> Self {
|
||||
match error {
|
||||
LoadError::Unsupported(error) => error.into(),
|
||||
LoadError::Ranks(message) => Self::Ranks(message),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,12 @@ pub trait Tokenizer: Send + Sync {
|
|||
fn count_tokens(&self, text: &str) -> Result<usize, Error>;
|
||||
}
|
||||
|
||||
pub trait TextCodec: Tokenizer {
|
||||
fn encode(&self, text: &str) -> Result<Vec<u32>, Error>;
|
||||
fn decode(&self, ids: &[u32], skip_special_tokens: bool) -> Result<String, Error>;
|
||||
fn name(&self) -> &str;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -58,7 +58,8 @@ from ._lazy_imports_registry import (
|
|||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
from tiktoken import Encoding
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
def get_litellm_globals() -> dict[str, object]:
|
||||
|
|
@ -89,26 +90,11 @@ def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "flo
|
|||
# These are special lazy loaders for things that are used internally
|
||||
# They're separate from the main lazy import system because they have specific use cases
|
||||
|
||||
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
|
||||
_default_encoding: "Encoding | None" = None
|
||||
|
||||
def _get_default_encoding() -> "Tokenizer":
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
|
||||
def _get_default_encoding() -> "Encoding":
|
||||
"""
|
||||
Lazily load and cache the default OpenAI encoding.
|
||||
|
||||
This avoids importing `litellm.litellm_core_utils.default_encoding` (and thus tiktoken)
|
||||
at `litellm` import time. The encoding is cached after the first import.
|
||||
|
||||
This is used internally by utils.py functions that need the encoding but shouldn't
|
||||
trigger its import during module load.
|
||||
"""
|
||||
global _default_encoding
|
||||
if _default_encoding is None:
|
||||
from litellm.litellm_core_utils.default_encoding import encoding
|
||||
|
||||
_default_encoding = encoding
|
||||
return _default_encoding
|
||||
return get_encoding("cl100k_base")
|
||||
|
||||
|
||||
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": "compact-2026-01-12",
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
_get_service_tier_cost_key,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_token,
|
||||
get_batch_cost_rates,
|
||||
get_billable_input_tokens,
|
||||
get_token_type_cost_breakdown,
|
||||
parse_prompt_tokens_details,
|
||||
|
|
@ -115,6 +116,7 @@ from litellm.types.utils import (
|
|||
LlmProviders,
|
||||
LlmProvidersSet,
|
||||
ModelInfo,
|
||||
ModelInfoBase,
|
||||
PromptTokensDetailsWrapper,
|
||||
ServiceTier,
|
||||
StandardBuiltInToolsParams,
|
||||
|
|
@ -322,6 +324,48 @@ class OCRPricing(TypedDict, total=False):
|
|||
annotation_cost_per_page: ReadOnly[float | None]
|
||||
|
||||
|
||||
_WALL_CLOCK_PRICED_MODES: Final = frozenset({"chat", "completion", "embedding", "responses"})
|
||||
|
||||
|
||||
def _has_token_or_tiered_pricing(model_info: ModelInfoBase) -> bool:
|
||||
return (
|
||||
(model_info.get("input_cost_per_token") or 0.0) > 0
|
||||
or (model_info.get("output_cost_per_token") or 0.0) > 0
|
||||
or model_info.get("tiered_pricing") is not None
|
||||
)
|
||||
|
||||
|
||||
def _bills_wall_clock_seconds(model_info: ModelInfoBase) -> bool:
|
||||
mode: Final = model_info.get("mode")
|
||||
return mode is None or mode in _WALL_CLOCK_PRICED_MODES
|
||||
|
||||
|
||||
def _per_second_pricing_cost(
|
||||
model: str,
|
||||
custom_llm_provider: str | None,
|
||||
response_time_ms: float | None,
|
||||
) -> tuple[float, float] | None:
|
||||
try:
|
||||
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
except Exception: # noqa: BLE001 # the lookup raises plain Exception for an unmapped model
|
||||
return None
|
||||
if _has_token_or_tiered_pricing(model_info) or not _bills_wall_clock_seconds(model_info):
|
||||
return None
|
||||
input_cost_per_second: Final = model_info.get("input_cost_per_second")
|
||||
output_cost_per_second: Final = model_info.get("output_cost_per_second")
|
||||
if input_cost_per_second is None and output_cost_per_second is None:
|
||||
return None
|
||||
seconds: Final = (response_time_ms or 0.0) / 1000
|
||||
verbose_logger.debug(
|
||||
"For model=%s - input_cost_per_second: %s; output_cost_per_second: %s; response time: %s",
|
||||
model,
|
||||
input_cost_per_second,
|
||||
output_cost_per_second,
|
||||
response_time_ms,
|
||||
)
|
||||
return (input_cost_per_second or 0.0) * seconds, (output_cost_per_second or 0.0) * seconds
|
||||
|
||||
|
||||
def cost_per_token(
|
||||
model: str = "",
|
||||
prompt_tokens: int = 0,
|
||||
|
|
@ -448,9 +492,6 @@ def cost_per_token(
|
|||
if response_cost is not None:
|
||||
return response_cost[0], response_cost[1]
|
||||
|
||||
# given
|
||||
prompt_tokens_cost_usd_dollar: float = 0
|
||||
completion_tokens_cost_usd_dollar: float = 0
|
||||
model_cost_ref: Final = litellm.model_cost
|
||||
# Only callers that explicitly pass `custom_llm_provider` get the
|
||||
# dedup/prefix-join treatment. When provider is omitted, preserve legacy
|
||||
|
|
@ -611,6 +652,14 @@ def cost_per_token(
|
|||
number_of_queries=number_of_queries or 1,
|
||||
optional_params=(getattr(response, "_hidden_params", None) if response else None),
|
||||
)
|
||||
elif (
|
||||
per_second_cost := _per_second_pricing_cost(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
response_time_ms=response_time_ms,
|
||||
)
|
||||
) is not None:
|
||||
return per_second_cost
|
||||
elif custom_llm_provider == "vertex_ai":
|
||||
cost_router: Final = google_cost_router(
|
||||
model=model_without_prefix,
|
||||
|
|
@ -685,12 +734,7 @@ def cost_per_token(
|
|||
)
|
||||
else:
|
||||
model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
|
||||
|
||||
if (
|
||||
(model_info.get("input_cost_per_token") or 0.0) > 0
|
||||
or (model_info.get("output_cost_per_token") or 0.0) > 0
|
||||
or model_info.get("tiered_pricing") is not None
|
||||
):
|
||||
if _has_token_or_tiered_pricing(model_info):
|
||||
return generic_cost_per_token(
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
|
|
@ -698,36 +742,8 @@ def cost_per_token(
|
|||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
)
|
||||
|
||||
input_cost_per_second: Final = model_info.get("input_cost_per_second")
|
||||
if input_cost_per_second is not None and response_time_ms is not None:
|
||||
verbose_logger.debug(
|
||||
"For model=%s - input_cost_per_second: %s; response time: %s",
|
||||
model,
|
||||
input_cost_per_second,
|
||||
response_time_ms,
|
||||
)
|
||||
## COST PER SECOND ##
|
||||
prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000
|
||||
|
||||
output_cost_per_second: Final = model_info.get("output_cost_per_second")
|
||||
if output_cost_per_second is not None and response_time_ms is not None:
|
||||
verbose_logger.debug(
|
||||
"For model=%s - output_cost_per_second: %s; response time: %s",
|
||||
model,
|
||||
output_cost_per_second,
|
||||
response_time_ms,
|
||||
)
|
||||
## COST PER SECOND ##
|
||||
completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000
|
||||
|
||||
verbose_logger.debug(
|
||||
"Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
|
||||
model,
|
||||
prompt_tokens_cost_usd_dollar,
|
||||
completion_tokens_cost_usd_dollar,
|
||||
)
|
||||
return prompt_tokens_cost_usd_dollar, completion_tokens_cost_usd_dollar
|
||||
verbose_logger.debug("No per-token, tiered, or per-second pricing for model=%s; cost is 0", model)
|
||||
return 0.0, 0.0
|
||||
|
||||
|
||||
def get_replicate_completion_pricing(completion_response: dict, total_time=0.0):
|
||||
|
|
@ -1222,6 +1238,21 @@ def _split_responses_ws_logging_object_by_service_tier(
|
|||
)
|
||||
|
||||
|
||||
def _response_time_ms_for_cost(
|
||||
completion_response: object,
|
||||
litellm_logging_obj: LitellmLoggingObject | None,
|
||||
total_time: float | None,
|
||||
) -> float:
|
||||
stamped: Final = getattr(completion_response, "_response_ms", None)
|
||||
if isinstance(stamped, (int, float)):
|
||||
return float(stamped)
|
||||
if total_time:
|
||||
return total_time
|
||||
if litellm_logging_obj is not None:
|
||||
return litellm_logging_obj.get_response_ms()
|
||||
return 0.0
|
||||
|
||||
|
||||
def completion_cost(
|
||||
completion_response: object | None = None,
|
||||
model: str | None = None,
|
||||
|
|
@ -1443,8 +1474,6 @@ def completion_cost(
|
|||
prompt_tokens_details = _usage.get("prompt_tokens_details") or {}
|
||||
cache_read_input_tokens = prompt_tokens_details.get("cached_tokens", 0)
|
||||
|
||||
total_time = getattr(completion_response, "_response_ms", 0)
|
||||
|
||||
hidden_params = getattr(completion_response, "_hidden_params", None)
|
||||
if hidden_params is not None:
|
||||
custom_llm_provider = hidden_params.get("custom_llm_provider", custom_llm_provider or None)
|
||||
|
|
@ -1676,6 +1705,11 @@ def completion_cost(
|
|||
)
|
||||
|
||||
return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
|
||||
response_time_ms = _response_time_ms_for_cost(
|
||||
completion_response=completion_response,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
total_time=total_time,
|
||||
)
|
||||
# Calculate cost based on prompt_tokens, completion_tokens
|
||||
if (
|
||||
"togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
|
||||
|
|
@ -1686,7 +1720,7 @@ def completion_cost(
|
|||
# see https://replicate.com/pricing
|
||||
elif (model in litellm.replicate_models or "replicate" in model) and model not in litellm.model_cost:
|
||||
# for unmapped replicate model, default to replicate's time tracking logic
|
||||
return get_replicate_completion_pricing(completion_response, total_time)
|
||||
return get_replicate_completion_pricing(completion_response, response_time_ms)
|
||||
|
||||
if model is None:
|
||||
raise ValueError(
|
||||
|
|
@ -1718,7 +1752,7 @@ def completion_cost(
|
|||
prompt_tokens=prompt_tokens or 0,
|
||||
completion_tokens=completion_tokens or 0,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
response_time_ms=total_time,
|
||||
response_time_ms=response_time_ms,
|
||||
region_name=None if explicit_pricing else region_name,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
|
|
@ -2524,35 +2558,14 @@ def batch_cost_calculator(
|
|||
if not model_info:
|
||||
return 0.0, 0.0
|
||||
|
||||
input_cost_per_token_batches: Final = model_info.get("input_cost_per_token_batches")
|
||||
batch_rates: Final = get_batch_cost_rates(model_info, usage, custom_llm_provider)
|
||||
input_cost_per_token: Final = model_info.get("input_cost_per_token")
|
||||
output_cost_per_token_batches: Final = model_info.get("output_cost_per_token_batches")
|
||||
output_cost_per_token: Final = model_info.get("output_cost_per_token")
|
||||
total_prompt_cost = 0.0
|
||||
total_completion_cost = 0.0
|
||||
if input_cost_per_token_batches is not None:
|
||||
batch_details: Final = parse_prompt_tokens_details(usage)
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
batch_details["audio_tokens"],
|
||||
batch_details["image_tokens"],
|
||||
batch_details["video_tokens"],
|
||||
)
|
||||
modality_rates: Final = (
|
||||
_batch_rate(model_info, "input_cost_per_audio_token_batches", input_cost_per_token_batches),
|
||||
_batch_rate(model_info, "input_cost_per_image_token_batches", input_cost_per_token_batches),
|
||||
_batch_rate(model_info, "input_cost_per_video_token_batches", input_cost_per_token_batches),
|
||||
)
|
||||
total_prompt_cost = sum(
|
||||
tokens * rate
|
||||
for tokens, rate in zip(
|
||||
(
|
||||
max((usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens, 0),
|
||||
audio_tokens,
|
||||
image_tokens,
|
||||
video_tokens,
|
||||
),
|
||||
(input_cost_per_token_batches, *modality_rates),
|
||||
)
|
||||
if batch_rates.input is not None:
|
||||
total_prompt_cost = _batch_prompt_cost(
|
||||
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
|
||||
)
|
||||
elif input_cost_per_token:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
|
|
@ -2572,8 +2585,8 @@ def batch_cost_calculator(
|
|||
|
||||
cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
|
||||
total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
|
||||
if output_cost_per_token_batches is not None:
|
||||
total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
|
||||
if batch_rates.output is not None:
|
||||
total_completion_cost = usage.completion_tokens * batch_rates.output
|
||||
elif output_cost_per_token:
|
||||
total_completion_cost = (
|
||||
usage.completion_tokens * (output_cost_per_token) / 2
|
||||
|
|
@ -2587,6 +2600,34 @@ def batch_cost_calculator(
|
|||
return total_prompt_cost, total_completion_cost
|
||||
|
||||
|
||||
def _batch_prompt_cost(
|
||||
usage: Usage,
|
||||
model_info: ModelInfo,
|
||||
input_rate: float,
|
||||
cache_read_rate: float | None,
|
||||
cache_creation_rate: float | None,
|
||||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
details["audio_tokens"],
|
||||
details["image_tokens"],
|
||||
details["video_tokens"],
|
||||
)
|
||||
text_tokens: Final = max(
|
||||
(usage.prompt_tokens or 0) - audio_tokens - image_tokens - video_tokens - cached_tokens - written_tokens, 0
|
||||
)
|
||||
return (
|
||||
text_tokens * input_rate
|
||||
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ cached_tokens * (cache_read_rate or 0.0)
|
||||
+ written_tokens * (cache_creation_rate or 0.0)
|
||||
)
|
||||
|
||||
|
||||
def _attribute_value(obj: object, name: str) -> object:
|
||||
return getattr(obj, name)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,13 +4,17 @@ arize AI is OTEL compatible
|
|||
this file has Arize ai specific helper functions
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
from collections.abc import Callable, Mapping
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.arize import _utils
|
||||
from litellm.integrations.arize._utils import ArizeOTELAttributes
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
from litellm.integrations.opentelemetry import _MAX_DYNAMIC_TRACER_PROVIDERS, OpenTelemetry, OpenTelemetryConfig
|
||||
from litellm.types.integrations.arize import ArizeConfig
|
||||
from litellm.types.services import ServiceLoggerPayload
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
|
@ -26,6 +30,9 @@ else:
|
|||
Protocol = Any
|
||||
Span = Any
|
||||
|
||||
_SUCCESS_SAMPLING_RATE_VAR: Final = "arize_success_sampling_rate"
|
||||
_ERROR_SAMPLING_RATE_VAR: Final = "arize_error_sampling_rate"
|
||||
|
||||
|
||||
class ArizeLogger(OpenTelemetry):
|
||||
"""
|
||||
|
|
@ -36,6 +43,26 @@ class ArizeLogger(OpenTelemetry):
|
|||
fighting over the global ``opentelemetry.trace`` TracerProvider singleton.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: OpenTelemetryConfig | None = None,
|
||||
callback_name: str | None = None,
|
||||
tracer_provider: object | None = None,
|
||||
logger_provider: object | None = None,
|
||||
meter_provider: object | None = None,
|
||||
max_dynamic_tracer_providers: int = _MAX_DYNAMIC_TRACER_PROVIDERS,
|
||||
random_draw: Callable[[], float] | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
config=config,
|
||||
callback_name=callback_name,
|
||||
tracer_provider=tracer_provider,
|
||||
logger_provider=logger_provider,
|
||||
meter_provider=meter_provider,
|
||||
max_dynamic_tracer_providers=max_dynamic_tracer_providers,
|
||||
)
|
||||
self._random_draw: Final[Callable[[], float]] = random_draw if random_draw is not None else random.random
|
||||
|
||||
def _init_tracing(self, tracer_provider):
|
||||
"""
|
||||
Override to always create a *private* TracerProvider for Arize.
|
||||
|
|
@ -55,6 +82,72 @@ class ArizeLogger(OpenTelemetry):
|
|||
self.tracer = provider.get_tracer("litellm")
|
||||
self.span_kind = SpanKind
|
||||
|
||||
def _handle_success(
|
||||
self,
|
||||
kwargs: dict[str, object],
|
||||
response_obj: object,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
) -> None:
|
||||
if not self._should_export(kwargs, _SUCCESS_SAMPLING_RATE_VAR):
|
||||
return
|
||||
super()._handle_success(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
def _handle_failure(
|
||||
self,
|
||||
kwargs: dict[str, object],
|
||||
response_obj: object,
|
||||
start_time: datetime,
|
||||
end_time: datetime,
|
||||
) -> None:
|
||||
if not self._should_export(kwargs, _ERROR_SAMPLING_RATE_VAR):
|
||||
return
|
||||
super()._handle_failure(kwargs, response_obj, start_time, end_time)
|
||||
|
||||
def _sampling_rate_for_request(self, kwargs: Mapping[str, object], var: str) -> float | None:
|
||||
dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
|
||||
if not isinstance(dynamic_params, Mapping):
|
||||
return None
|
||||
value: Final = dynamic_params.get(var)
|
||||
if value is None or value in ("", "None"):
|
||||
return None
|
||||
try:
|
||||
rate = float(value)
|
||||
except (TypeError, ValueError):
|
||||
verbose_logger.warning(
|
||||
"ArizeLogger: %s value %r is not a number; exporting the request",
|
||||
var,
|
||||
value,
|
||||
)
|
||||
return None
|
||||
if not math.isfinite(rate) or not 0.0 <= rate <= 1.0:
|
||||
verbose_logger.warning(
|
||||
"ArizeLogger: %s value %r is outside 0.0..1.0; exporting the request",
|
||||
var,
|
||||
value,
|
||||
)
|
||||
return None
|
||||
return rate
|
||||
|
||||
def _should_export(self, kwargs: dict[str, object], var: str) -> bool:
|
||||
rate: Final = self._sampling_rate_for_request(kwargs, var)
|
||||
if rate is None:
|
||||
return True
|
||||
otel_internal: Final = self._otel_internal_state(kwargs)
|
||||
key: Final = f"arize_sampled:{var}"
|
||||
cached: Final = otel_internal.get(key)
|
||||
if isinstance(cached, bool):
|
||||
return cached
|
||||
sampled: Final = rate > 0.0 and self._random_draw() <= rate
|
||||
otel_internal[key] = sampled
|
||||
if not sampled:
|
||||
verbose_logger.debug(
|
||||
"ArizeLogger: dropping request, %s rate %r rejected the draw",
|
||||
var,
|
||||
rate,
|
||||
)
|
||||
return sampled
|
||||
|
||||
def _init_otel_logger_on_litellm_proxy(self):
|
||||
"""
|
||||
Override: Arize should NOT overwrite the proxy's
|
||||
|
|
|
|||
|
|
@ -1240,6 +1240,25 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
# End of Team/Key Based Logging Control Flow
|
||||
#########################################################
|
||||
|
||||
def _otel_internal_state(self, kwargs: dict[str, object]) -> dict[str, object]:
|
||||
"""Return the request-local ``_otel_internal`` marker dict, creating it if absent."""
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if not isinstance(litellm_params, dict):
|
||||
litellm_params = {}
|
||||
kwargs["litellm_params"] = litellm_params
|
||||
|
||||
_metadata = litellm_params.get("metadata")
|
||||
if not isinstance(_metadata, dict):
|
||||
_metadata = {}
|
||||
litellm_params["metadata"] = _metadata
|
||||
|
||||
_otel_internal = _metadata.get("_otel_internal")
|
||||
if not isinstance(_otel_internal, dict):
|
||||
_otel_internal = {}
|
||||
_metadata["_otel_internal"] = _otel_internal
|
||||
|
||||
return _otel_internal
|
||||
|
||||
def _emit_once(self, kwargs: dict, *scope: object) -> bool:
|
||||
"""Return True the first time this handler is asked to emit a span
|
||||
for the given (handler, scope) on this kwargs; False on repeats.
|
||||
|
|
@ -1264,20 +1283,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
|
|||
request-local (kwargs is shared across the sync/async callbacks and
|
||||
lifecycle hooks for one request).
|
||||
"""
|
||||
litellm_params = kwargs.get("litellm_params")
|
||||
if not isinstance(litellm_params, dict):
|
||||
litellm_params = {}
|
||||
kwargs["litellm_params"] = litellm_params
|
||||
|
||||
_metadata = litellm_params.get("metadata")
|
||||
if not isinstance(_metadata, dict):
|
||||
_metadata = {}
|
||||
litellm_params["metadata"] = _metadata
|
||||
|
||||
_otel_internal = _metadata.get("_otel_internal")
|
||||
if not isinstance(_otel_internal, dict):
|
||||
_otel_internal = {}
|
||||
_metadata["_otel_internal"] = _otel_internal
|
||||
_otel_internal = self._otel_internal_state(kwargs)
|
||||
|
||||
spans_logged = _otel_internal.get("spans_logged")
|
||||
if not isinstance(spans_logged, dict):
|
||||
|
|
|
|||
|
|
@ -427,7 +427,7 @@ class LLMCallSpanData:
|
|||
# plain ``.get`` — no repeated ``isinstance`` guards.
|
||||
raw_response: Final = payload.get("response")
|
||||
response: Final = cast(Mapping[str, object], raw_response if isinstance(raw_response, dict) else {})
|
||||
choices_out: Final = _dicts(response.get("choices")) or _responses_choices(response) or _ocr_choices(response)
|
||||
choices_out: Final = _output_choices(response)
|
||||
# ``finish_reasons`` is metadata, not content, so derive it from
|
||||
# ``choices_out`` before gating. The raw message/choice bodies are only
|
||||
# retained when content capture is enabled (see ``capture_span_content``);
|
||||
|
|
@ -752,20 +752,127 @@ def _responses_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
|||
return (choice,)
|
||||
|
||||
|
||||
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
markdowns: Final = tuple(
|
||||
text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None
|
||||
def _output_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
"""The response output as chat-shaped choices; images and binary bodies become size summaries, never bytes."""
|
||||
return (
|
||||
_completion_choices(response)
|
||||
or _responses_choices(response)
|
||||
or _ocr_choices(response)
|
||||
or _transcription_choices(response)
|
||||
or _moderation_choices(response)
|
||||
or _rerank_choices(response)
|
||||
or _search_choices(response)
|
||||
or _image_choices(response)
|
||||
or _binary_choices(response)
|
||||
)
|
||||
if not markdowns:
|
||||
|
||||
|
||||
def _text_choice(content: str, finish_reason: str | None = None) -> _Choice:
|
||||
message: Final[_AssistantMessage] = {"role": "assistant", "content": content, "refusal": None, "tool_calls": None}
|
||||
return {"message": message, "finish_reason": finish_reason}
|
||||
|
||||
|
||||
def _joined_choice(parts: tuple[str, ...]) -> tuple[_Choice, ...]:
|
||||
return (_text_choice("\n\n".join(parts)),) if parts else ()
|
||||
|
||||
|
||||
def _completion_choices(response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
return tuple(
|
||||
_text_choice(text, as_str(choice.get("finish_reason")))
|
||||
if "message" not in choice and isinstance(text := choice.get("text"), str)
|
||||
else choice
|
||||
for choice in _dicts(response.get("choices"))
|
||||
)
|
||||
|
||||
|
||||
def _ocr_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(text for page in _dicts(response.get("pages")) if (text := as_str(page.get("markdown"))) is not None)
|
||||
)
|
||||
|
||||
|
||||
def _transcription_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
text: Final = response.get("text")
|
||||
return (_text_choice(text),) if isinstance(text, str) and text else ()
|
||||
|
||||
|
||||
def _moderation_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(
|
||||
_moderation_verdict(flagged, result.get("categories"))
|
||||
for result in _dicts(response.get("results"))
|
||||
if isinstance(flagged := result.get("flagged"), bool)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _moderation_verdict(flagged: bool, categories: object) -> str:
|
||||
if not flagged:
|
||||
return "not flagged"
|
||||
hits: Final = (
|
||||
tuple(name for name, hit in cast(Mapping[str, object], categories).items() if hit is True)
|
||||
if isinstance(categories, dict)
|
||||
else ()
|
||||
)
|
||||
return f"flagged: {', '.join(hits)}" if hits else "flagged"
|
||||
|
||||
|
||||
def _rerank_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(
|
||||
_rerank_line(index, score, result.get("document"))
|
||||
for result in _dicts(response.get("results"))
|
||||
if (index := as_int(result.get("index"))) is not None
|
||||
if (score := as_float(result.get("relevance_score"))) is not None
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _rerank_line(index: int, score: float, document: object) -> str:
|
||||
text: Final = as_str((as_str_mapping(document) or {}).get("text"))
|
||||
return f"[{index}] {score}\n{text}" if text else f"[{index}] {score}"
|
||||
|
||||
|
||||
def _search_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(
|
||||
line
|
||||
for result in _dicts(response.get("results"))
|
||||
if (line := "\n".join(part for key in ("title", "url", "snippet") if (part := as_str(result.get(key)))))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _image_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
return _joined_choice(
|
||||
tuple(summary for item in _dicts(response.get("data")) if (summary := _image_summary(item)) is not None)
|
||||
)
|
||||
|
||||
|
||||
def _image_summary(item: Mapping[str, object]) -> str | None:
|
||||
location: Final = _image_location(item)
|
||||
if location is None:
|
||||
return None
|
||||
revised: Final = as_str(item.get("revised_prompt"))
|
||||
return f"{revised}\n{location}" if revised else location
|
||||
|
||||
|
||||
def _image_location(item: Mapping[str, object]) -> str | None:
|
||||
url: Final = as_str(item.get("url"))
|
||||
if url is not None:
|
||||
return url
|
||||
encoded: Final = item.get("b64_json")
|
||||
if not isinstance(encoded, str):
|
||||
return None
|
||||
return f"b64_json image ({len(encoded) * 3 // 4 - encoded[-2:].count('=')} bytes)"
|
||||
|
||||
|
||||
def _binary_choices(response: Mapping[str, object]) -> tuple[_Choice, ...]:
|
||||
size: Final = as_int(response.get("num_bytes"))
|
||||
if size is None:
|
||||
return ()
|
||||
message: Final[_AssistantMessage] = {
|
||||
"role": "assistant",
|
||||
"content": "\n\n".join(markdowns),
|
||||
"refusal": None,
|
||||
"tool_calls": None,
|
||||
}
|
||||
choice: Final[_Choice] = {"message": message, "finish_reason": None}
|
||||
return (choice,)
|
||||
content_type: Final = as_str(response.get("content_type"))
|
||||
return (_text_choice(f"{content_type} ({size} bytes)" if content_type else f"{size} bytes"),)
|
||||
|
||||
|
||||
def _responses_parts_text(parts: tuple[Mapping[str, object], ...], part_type: str, field: str) -> str | None:
|
||||
|
|
|
|||
|
|
@ -6,8 +6,9 @@ Core files:
|
|||
- `streaming_handler.py`: The core streaming logic + streaming related helper utils
|
||||
- `core_helpers.py`: code used in `types/` - e.g. `map_finish_reason`.
|
||||
- `exception_mapping_utils.py`: utils for mapping exceptions to openai-compatible error types.
|
||||
- `default_encoding.py`: code for loading the default encoding (tiktoken)
|
||||
- `default_encoding.py`: code for loading the default Python tokenizer and bundled cache
|
||||
- `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name.
|
||||
- `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
|
||||
- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
|
||||
|
||||
Tokenizer factories return Python tokenizer objects by default. Set `LITELLM_RUST=1` or call `litellm.rust(True)` before constructing tokenizers to select the Rust backend through `Route.TOKENIZER` in the Rust catalog. Missing native bindings or unsupported native features fall back to Python. Existing tokenizer objects keep their selected backend. Rust-backed tokenizer objects carry the read-only `tiktoken.Encoding` / `tokenizers.Tokenizer` surface and are immutable: `enable_padding`, `enable_truncation` and `add_tokens` stay on the Python tokenizer.
|
||||
|
|
|
|||
180
litellm/litellm_core_utils/bug_report.py
Normal file
180
litellm/litellm_core_utils/bug_report.py
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import platform
|
||||
import traceback
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import litellm
|
||||
from litellm._version import version as litellm_version
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
ISSUE_URL_BASE: Final = "https://github.com/BerriAI/litellm/issues/new"
|
||||
MAX_URL_LENGTH: Final = 6000
|
||||
MAX_FRAMES: Final = 12
|
||||
DISABLE_ENV_VAR: Final = "LITELLM_DISABLE_BUG_REPORT_LINK"
|
||||
NOTICE_PREFIX: Final = "This looks like a bug in LiteLLM rather than in your request."
|
||||
KNOWN_PROVIDERS: Final = frozenset(provider.value for provider in LlmProviders)
|
||||
|
||||
Surface = Literal["sdk", "proxy"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BugReport:
|
||||
surface: Surface
|
||||
exception_type: str
|
||||
litellm_frames: tuple[str, ...]
|
||||
litellm_version: str
|
||||
python_version: str
|
||||
call_type: str | None
|
||||
custom_llm_provider: str | None
|
||||
stream: bool | None
|
||||
config_lines: tuple[str, ...]
|
||||
|
||||
|
||||
def bug_report_enabled() -> bool:
|
||||
return os.getenv(DISABLE_ENV_VAR, "").lower() != "true"
|
||||
|
||||
|
||||
def should_report_bug(exc: object) -> bool:
|
||||
return bug_report_enabled() and isinstance(exc, BaseException) and getattr(exc, "status_code", None) is None
|
||||
|
||||
|
||||
def _format_frame(frame: traceback.FrameSummary, package_dir: Path, package_parent: Path) -> str | None:
|
||||
frame_path: Final = Path(frame.filename).resolve()
|
||||
try:
|
||||
frame_path.relative_to(package_dir)
|
||||
relative_path: Final = frame_path.relative_to(package_parent)
|
||||
except ValueError:
|
||||
return None
|
||||
return f"{relative_path.as_posix()}:{frame.lineno} in {frame.name}"
|
||||
|
||||
|
||||
def _get_litellm_frames(exc: BaseException) -> tuple[str, ...]:
|
||||
if exc.__traceback__ is None:
|
||||
return ()
|
||||
package_dir: Final = Path(litellm.__file__).resolve().parent
|
||||
package_parent: Final = package_dir.parent
|
||||
return tuple(
|
||||
frame_text
|
||||
for frame in traceback.extract_tb(exc.__traceback__)
|
||||
if (frame_text := _format_frame(frame, package_dir, package_parent)) is not None
|
||||
)[-MAX_FRAMES:]
|
||||
|
||||
|
||||
def allowlisted(value: object, allowed: frozenset[str]) -> str | None:
|
||||
return value if isinstance(value, str) and value in allowed else None
|
||||
|
||||
|
||||
def build_bug_report(
|
||||
exc: BaseException,
|
||||
*,
|
||||
surface: Surface,
|
||||
call_type: str | None = None,
|
||||
custom_llm_provider: object = None,
|
||||
stream: object = None,
|
||||
config_lines: tuple[str, ...] = (),
|
||||
) -> BugReport:
|
||||
return BugReport(
|
||||
surface=surface,
|
||||
exception_type=type(exc).__name__,
|
||||
litellm_frames=_get_litellm_frames(exc),
|
||||
litellm_version=litellm_version,
|
||||
python_version=platform.python_version(),
|
||||
call_type=call_type,
|
||||
custom_llm_provider=allowlisted(custom_llm_provider, KNOWN_PROVIDERS),
|
||||
stream=stream if isinstance(stream, bool) else None,
|
||||
config_lines=config_lines,
|
||||
)
|
||||
|
||||
|
||||
def _domain(report: BugReport) -> str:
|
||||
if report.surface == "sdk":
|
||||
return "Python SDK: the litellm package itself"
|
||||
if report.custom_llm_provider is not None:
|
||||
return "LLM translation: a specific provider's request or response"
|
||||
return "Proxy core: startup, config, health checks, endpoints"
|
||||
|
||||
|
||||
def _title(report: BugReport, frames: tuple[str, ...]) -> str:
|
||||
location: Final = frames[-1].split(":", 1)[0] if frames else "litellm"
|
||||
return f"[Bug]: {report.exception_type} in {location}"
|
||||
|
||||
|
||||
def _description(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
|
||||
frame_block: Final = "LiteLLM frames:\n```\n" + "\n".join(frames) + "\n```\n\n" if frames else ""
|
||||
stream_line: Final = "" if report.stream is None else f"Stream: {str(report.stream).lower()}\n"
|
||||
config_block: Final = (
|
||||
"\nConfig (true/false flags and LiteLLM-defined values only):\n```\n" + "\n".join(config_lines) + "\n```\n"
|
||||
if config_lines
|
||||
else ""
|
||||
)
|
||||
return (
|
||||
"Auto-generated by LiteLLM's bug report link. It carries no request data or error text. "
|
||||
"Please describe what you were doing, and paste the error message from your log below "
|
||||
"if it contains nothing sensitive.\n\n"
|
||||
"```\n\n```\n\n"
|
||||
f"Exception: `{report.exception_type}`\n\n"
|
||||
f"{frame_block}"
|
||||
f"Surface: {report.surface}\n"
|
||||
f"Endpoint / call: {report.call_type or 'unknown'}\n"
|
||||
f"Provider: {report.custom_llm_provider or 'unknown'}\n"
|
||||
f"LiteLLM: {report.litellm_version}\n"
|
||||
f"Python: {report.python_version}\n"
|
||||
f"{stream_line}"
|
||||
f"{config_block}"
|
||||
)
|
||||
|
||||
|
||||
def _issue_url(report: BugReport, frames: tuple[str, ...], config_lines: tuple[str, ...]) -> str:
|
||||
deployment: Final[tuple[tuple[str, str], ...]] = (
|
||||
(("deployment", "pip / Python SDK"),)
|
||||
if report.surface == "sdk"
|
||||
else (("deployment", "Docker"),)
|
||||
if os.path.exists("/.dockerenv")
|
||||
else ()
|
||||
)
|
||||
fields: Final = (
|
||||
("template", "bug_report.yml"),
|
||||
("labels", "bug"),
|
||||
("title", _title(report, frames)),
|
||||
("version", report.litellm_version),
|
||||
("domain", _domain(report)),
|
||||
("description", _description(report, frames, config_lines)),
|
||||
) + deployment
|
||||
return f"{ISSUE_URL_BASE}?{urlencode(fields)}"
|
||||
|
||||
|
||||
def bug_report_issue_url(report: BugReport) -> str:
|
||||
frames: Final = report.litellm_frames
|
||||
config_lines: Final = report.config_lines
|
||||
candidates: Final = (
|
||||
*((frames, config_lines[:count]) for count in range(len(config_lines), -1, -1)),
|
||||
*((frames[index:], ()) for index in range(1, len(frames) + 1)),
|
||||
)
|
||||
return next(
|
||||
(
|
||||
url
|
||||
for candidate_frames, candidate_config in candidates
|
||||
if len(url := _issue_url(report, candidate_frames, candidate_config)) <= MAX_URL_LENGTH
|
||||
),
|
||||
_issue_url(report, (), ()),
|
||||
)
|
||||
|
||||
|
||||
def strip_bug_report_notice(message: str) -> str:
|
||||
index: Final = message.find(NOTICE_PREFIX)
|
||||
if index == -1:
|
||||
return message
|
||||
head: Final = message[:index]
|
||||
return head.removesuffix("\n")
|
||||
|
||||
|
||||
def bug_report_notice(report: BugReport) -> str:
|
||||
return (
|
||||
f"{NOTICE_PREFIX} File it with one click "
|
||||
f"(prefilled, no request data or error text, review before submitting): {bug_report_issue_url(report)}"
|
||||
)
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import litellm
|
||||
|
|
@ -15,20 +14,6 @@ except (ImportError, AttributeError):
|
|||
|
||||
filename = pkg_resources.resource_filename(__name__, "litellm_core_utils/tokenizers")
|
||||
|
||||
CL100K_BASE_RANK_FILE: Final = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
|
||||
O200K_BASE_RANK_FILE: Final = "fb374d419588a4632f3f557e76b4b70aebbca790"
|
||||
|
||||
|
||||
def cl100k_base_rank_file() -> str:
|
||||
"""The vendored tiktoken `cl100k_base` rank file (`base64(token) rank` lines)."""
|
||||
return Path(filename, CL100K_BASE_RANK_FILE).read_text(encoding="ascii")
|
||||
|
||||
|
||||
def o200k_base_rank_file() -> str:
|
||||
"""The vendored tiktoken `o200k_base` rank file (`base64(token) rank` lines)."""
|
||||
return Path(filename, O200K_BASE_RANK_FILE).read_text(encoding="ascii")
|
||||
|
||||
|
||||
# Always default TIKTOKEN_CACHE_DIR to the bundled tokenizers directory
|
||||
# unless the user explicitly overrides it via CUSTOM_TIKTOKEN_CACHE_DIR.
|
||||
# This keeps tiktoken fully offline-capable by default (see #1071).
|
||||
|
|
|
|||
|
|
@ -10,6 +10,11 @@ import httpx
|
|||
|
||||
import litellm
|
||||
from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
|
||||
from litellm.litellm_core_utils.bug_report import (
|
||||
bug_report_notice,
|
||||
build_bug_report,
|
||||
should_report_bug,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
|
|
@ -2673,7 +2678,21 @@ def exception_type(
|
|||
)
|
||||
else:
|
||||
raise APIConnectionError(
|
||||
message=f"{original_exception}\n{_redact_string(traceback.format_exc())}",
|
||||
message=(
|
||||
f"{original_exception}\n{_redact_string(traceback.format_exc())}"
|
||||
+ (
|
||||
"\n"
|
||||
+ bug_report_notice(
|
||||
build_bug_report(
|
||||
original_exception,
|
||||
surface="sdk",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
)
|
||||
if should_report_bug(original_exception)
|
||||
else ""
|
||||
)
|
||||
),
|
||||
llm_provider=custom_llm_provider,
|
||||
model=model,
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1/"), # stub the request
|
||||
|
|
|
|||
|
|
@ -1,10 +1,24 @@
|
|||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import Generator, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
|
||||
|
||||
_CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
|
||||
_inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
|
||||
"inherited_message_logging_disabled", default=False
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def inherit_message_logging_privacy(disabled: bool) -> Generator[None]:
|
||||
token: Final = _inherited_message_logging_disabled.set(_inherited_message_logging_disabled.get() or disabled)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_inherited_message_logging_disabled.reset(token)
|
||||
|
||||
|
||||
def iter_client_callback_metadata_dicts(
|
||||
|
|
@ -84,6 +98,8 @@ _supported_callback_params: Final[tuple[str, ...]] = (
|
|||
"arize_api_key",
|
||||
"arize_space_key",
|
||||
"arize_space_id",
|
||||
"arize_success_sampling_rate",
|
||||
"arize_error_sampling_rate",
|
||||
"posthog_api_key",
|
||||
"posthog_host",
|
||||
"braintrust_api_key",
|
||||
|
|
@ -143,7 +159,7 @@ def get_trusted_callback_params(kwargs: Mapping[str, Any] | None) -> tuple[tuple
|
|||
|
||||
|
||||
def initialize_standard_callback_dynamic_params(
|
||||
kwargs: dict | None = None,
|
||||
kwargs: dict[str, object] | None = None,
|
||||
) -> StandardCallbackDynamicParams:
|
||||
"""
|
||||
Initialize the standard callback dynamic params from the kwargs
|
||||
|
|
@ -179,4 +195,10 @@ def initialize_standard_callback_dynamic_params(
|
|||
if param in _trusted_overlay_callback_params:
|
||||
standard_callback_dynamic_params[param] = trusted_value
|
||||
|
||||
if _inherited_message_logging_disabled.get():
|
||||
private_params: Final[StandardCallbackDynamicParams] = {
|
||||
**standard_callback_dynamic_params,
|
||||
"turn_off_message_logging": True,
|
||||
}
|
||||
return private_params
|
||||
return standard_callback_dynamic_params
|
||||
|
|
|
|||
|
|
@ -107,6 +107,10 @@ from litellm.litellm_core_utils.redact_messages import (
|
|||
redact_streaming_responses_for_custom_logger,
|
||||
should_redact_message_logging,
|
||||
)
|
||||
from litellm.litellm_core_utils.served_output_texts import (
|
||||
SERVED_OUTPUT_TEXTS_KEY,
|
||||
overlay_served_output_texts,
|
||||
)
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
|
|
@ -383,11 +387,40 @@ _DEPLOYMENT_PRICING_KEYS: Final = (
|
|||
"output_cost_per_token",
|
||||
"input_cost_per_token_batches",
|
||||
"output_cost_per_token_batches",
|
||||
"input_cost_per_token_above_272k_tokens_batches",
|
||||
"output_cost_per_token_above_272k_tokens_batches",
|
||||
"cache_read_input_token_cost_batches",
|
||||
"cache_read_input_token_cost_above_272k_tokens_batches",
|
||||
"cache_creation_input_token_cost_batches",
|
||||
"cache_creation_input_token_cost_above_272k_tokens_batches",
|
||||
"ocr_cost_per_page",
|
||||
"ocr_cost_per_page_batches",
|
||||
"annotation_cost_per_page",
|
||||
"annotation_cost_per_page_batches",
|
||||
)
|
||||
_INPUT_PRICING_KEY_PREFIXES: Final = (
|
||||
"input_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
)
|
||||
_OUTPUT_PRICING_KEY_PREFIXES: Final = ("output_cost_per_token",)
|
||||
_BATCH_PRICING_KEY_SUFFIX: Final = "_batches"
|
||||
|
||||
|
||||
_NO_CARRIED_RATES: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
def _published_direction(
|
||||
published: ModelInfo, registered: Mapping[str, object], flat_key: str, prefixes: tuple[str, ...]
|
||||
) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in published.items()
|
||||
if registered.get(key) is None
|
||||
and (key == flat_key or (key.startswith(prefixes) and key.endswith(_BATCH_PRICING_KEY_SUFFIX)))
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def deployment_pricing_model_info(model_id: str | None, deployment_model: str | None) -> ModelInfo | None:
|
||||
|
|
@ -399,12 +432,15 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str |
|
|||
get_model_info fills absent costs with 0, so asking it directly cannot
|
||||
tell "configured as free" apart from "no pricing configured". A deployment
|
||||
may declare only one side of its pricing, so the side it leaves out keeps
|
||||
the model's published rates instead of billing as zero. Ownership is per
|
||||
token direction: declaring either rate for a direction takes that whole
|
||||
direction, so a published batch rate can never displace a standard rate
|
||||
the deployment configured itself. OCR per-page rates count as declared
|
||||
pricing too; they pass through as registered and ``ocr_batch_cost`` layers
|
||||
the published rate under each per-page family the deployment leaves out.
|
||||
the model's published standard rate and every published batch rate for
|
||||
that direction (flat, long-context tier, cached, cache write) instead of
|
||||
billing as zero. Ownership is per token direction: declaring the flat
|
||||
standard or flat batch rate for a direction takes that whole direction, so
|
||||
a published batch rate can never displace a standard rate the deployment
|
||||
configured itself. A tier-only override keeps every published rate it left
|
||||
out. OCR per-page rates count as declared pricing too; they pass through as
|
||||
registered and ``ocr_batch_cost`` layers the published rate under each
|
||||
per-page family the deployment leaves out.
|
||||
"""
|
||||
if model_id is None:
|
||||
return None
|
||||
|
|
@ -425,13 +461,22 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str |
|
|||
registered.get("output_cost_per_token") is not None
|
||||
or registered.get("output_cost_per_token_batches") is not None
|
||||
)
|
||||
if not declares_input:
|
||||
merged["input_cost_per_token"] = published.get("input_cost_per_token")
|
||||
merged["input_cost_per_token_batches"] = published.get("input_cost_per_token_batches")
|
||||
if not declares_output:
|
||||
merged["output_cost_per_token"] = published.get("output_cost_per_token")
|
||||
merged["output_cost_per_token_batches"] = published.get("output_cost_per_token_batches")
|
||||
return merged
|
||||
carried_input: Final = (
|
||||
_NO_CARRIED_RATES
|
||||
if declares_input
|
||||
else _published_direction(published, registered, "input_cost_per_token", _INPUT_PRICING_KEY_PREFIXES)
|
||||
)
|
||||
carried_output: Final = (
|
||||
_NO_CARRIED_RATES
|
||||
if declares_output
|
||||
else _published_direction(published, registered, "output_cost_per_token", _OUTPUT_PRICING_KEY_PREFIXES)
|
||||
)
|
||||
priced: Final[ModelInfo] = { # pyright: ignore[reportAssignmentType] # carried keys are ModelInfo rates
|
||||
**merged,
|
||||
**carried_input,
|
||||
**carried_output,
|
||||
}
|
||||
return priced
|
||||
|
||||
|
||||
def _published_pricing(deployment_model: str | None) -> ModelInfo | None:
|
||||
|
|
@ -496,6 +541,14 @@ def mask_api_base_credentials(api_base: str) -> str:
|
|||
return api_base[:key_end] + "*" * 5 + api_base[-4:]
|
||||
|
||||
|
||||
def _timestamp_seconds(moment: object) -> float | None:
|
||||
if isinstance(moment, datetime.datetime):
|
||||
return moment.timestamp()
|
||||
if isinstance(moment, (int, float)):
|
||||
return float(moment)
|
||||
return None
|
||||
|
||||
|
||||
class Logging(LiteLLMLoggingBaseClass):
|
||||
global \
|
||||
supabaseClient, \
|
||||
|
|
@ -1634,10 +1687,12 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
return response.mcp_tool_call_response
|
||||
|
||||
def get_response_ms(self) -> float:
|
||||
return (
|
||||
self.model_call_details.get("end_time", datetime.datetime.now())
|
||||
- self.model_call_details.get("start_time", datetime.datetime.now())
|
||||
).total_seconds() * 1000
|
||||
now: Final = datetime.datetime.now()
|
||||
start_seconds: Final = _timestamp_seconds(self.model_call_details.get("start_time", now))
|
||||
end_seconds: Final = _timestamp_seconds(self.model_call_details.get("end_time", now))
|
||||
if start_seconds is None or end_seconds is None:
|
||||
return 0.0
|
||||
return (end_seconds - start_seconds) * 1000
|
||||
|
||||
def set_cost_breakdown(
|
||||
self,
|
||||
|
|
@ -5833,15 +5888,12 @@ class StandardLoggingPayloadSetup:
|
|||
|
||||
modified_final_response_obj: Final = redact_message_input_output_from_logging(
|
||||
model_call_details=kwargs,
|
||||
result=final_response_obj,
|
||||
result=overlay_served_output_texts(final_response_obj, kwargs.get(SERVED_OUTPUT_TEXTS_KEY)),
|
||||
)
|
||||
|
||||
if modified_final_response_obj is not None and isinstance(modified_final_response_obj, BaseModel):
|
||||
final_response_obj = modified_final_response_obj.model_dump()
|
||||
else:
|
||||
final_response_obj = modified_final_response_obj
|
||||
|
||||
return final_response_obj
|
||||
return modified_final_response_obj.model_dump()
|
||||
return modified_final_response_obj
|
||||
|
||||
@staticmethod
|
||||
def get_additional_headers(
|
||||
|
|
@ -6277,6 +6329,8 @@ def _extract_response_obj_and_hidden_params(
|
|||
hidden_params = getattr(init_response_obj, "_hidden_params", None)
|
||||
elif isinstance(init_response_obj, dict):
|
||||
response_obj = init_response_obj
|
||||
elif isinstance(init_response_obj, HttpxBinaryResponseContent):
|
||||
response_obj = dict(init_response_obj.logging_summary())
|
||||
else:
|
||||
response_obj = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -67,6 +67,15 @@ _SERVICE_TIER_TO_COST_KEY_SUFFIX: Final[Mapping[str, str]] = MappingProxyType(
|
|||
)
|
||||
|
||||
_INCLUSIVE_THRESHOLD_PROVIDERS: Final = frozenset({"xai"})
|
||||
_BATCH_KEY_SUFFIX: Final = "_batches"
|
||||
_BATCH_RATE_PREFIXES: Final = (
|
||||
"input_cost_per_token",
|
||||
"output_cost_per_token",
|
||||
"cache_read_input_token_cost",
|
||||
"cache_creation_input_token_cost",
|
||||
)
|
||||
_BATCH_TIER_KEY: Final = re.compile(rf"^({'|'.join(_BATCH_RATE_PREFIXES)})_above_(\d+k?)_tokens{_BATCH_KEY_SUFFIX}$")
|
||||
_NON_STANDARD_THRESHOLD_SUFFIXES: Final = (*_SERVICE_TIER_SUFFIXES, _BATCH_KEY_SUFFIX)
|
||||
|
||||
|
||||
def _uses_inclusive_token_thresholds(custom_llm_provider: str | None) -> bool:
|
||||
|
|
@ -248,9 +257,80 @@ def _get_service_tier_cost_key(base_key: str, service_tier: str | None) -> str:
|
|||
return f"{base_key}_{suffix}"
|
||||
|
||||
|
||||
def _parse_token_threshold(threshold: str) -> float:
|
||||
return float(threshold.replace("k", "")) * (1000 if "k" in threshold else 1)
|
||||
|
||||
|
||||
def _parse_above_token_threshold(key: str) -> float:
|
||||
threshold_str: Final = key.split("_above_")[1].split("_tokens")[0]
|
||||
return float(threshold_str.replace("k", "")) * (1000 if "k" in threshold_str else 1)
|
||||
return _parse_token_threshold(key.split("_above_")[1].split("_tokens")[0])
|
||||
|
||||
|
||||
def _prompt_exceeds_threshold(prompt_tokens: int, threshold: float, inclusive: bool) -> bool:
|
||||
return prompt_tokens > threshold or (inclusive and prompt_tokens == threshold)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BatchCostRates:
|
||||
input: float | None
|
||||
output: float | None
|
||||
cache_read: float | None
|
||||
cache_creation: float | None
|
||||
|
||||
|
||||
def _batch_rate(model_info: ModelInfo, key: str) -> float | None:
|
||||
value: Final = model_info.get(key)
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _batch_tier_rate(model_info: ModelInfo, tier_key: str, flat_key: str) -> float | None:
|
||||
tier_rate: Final = _batch_rate(model_info, tier_key)
|
||||
return _batch_rate(model_info, flat_key) if tier_rate is None else tier_rate
|
||||
|
||||
|
||||
def _batch_tier_thresholds(model_info: ModelInfo, prefix: str) -> frozenset[str]:
|
||||
return frozenset(
|
||||
tier.group(2)
|
||||
for key, value in model_info.items()
|
||||
if value is not None and (tier := _BATCH_TIER_KEY.match(key)) is not None and tier.group(1) == prefix
|
||||
)
|
||||
|
||||
|
||||
def _crossed_batch_tier(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> str | None:
|
||||
return next(
|
||||
(
|
||||
threshold
|
||||
for threshold in sorted(
|
||||
_batch_tier_thresholds(model_info, prefix), key=_parse_token_threshold, reverse=True
|
||||
)
|
||||
if _prompt_exceeds_threshold(usage.prompt_tokens, _parse_token_threshold(threshold), inclusive)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _batch_rate_for_prefix(model_info: ModelInfo, prefix: str, usage: Usage, inclusive: bool) -> float | None:
|
||||
flat_key: Final = f"{prefix}{_BATCH_KEY_SUFFIX}"
|
||||
threshold: Final = _crossed_batch_tier(model_info, prefix, usage, inclusive)
|
||||
if threshold is None:
|
||||
return _batch_rate(model_info, flat_key)
|
||||
return _batch_tier_rate(model_info, f"{prefix}_above_{threshold}_tokens{_BATCH_KEY_SUFFIX}", flat_key)
|
||||
|
||||
|
||||
def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provider: str | None) -> BatchCostRates:
|
||||
inclusive: Final = _uses_inclusive_token_thresholds(custom_llm_provider)
|
||||
return BatchCostRates(
|
||||
input=_batch_rate_for_prefix(model_info, "input_cost_per_token", usage, inclusive),
|
||||
output=_batch_rate_for_prefix(model_info, "output_cost_per_token", usage, inclusive),
|
||||
cache_read=_batch_rate_for_prefix(model_info, "cache_read_input_token_cost", usage, inclusive),
|
||||
cache_creation=_batch_rate_for_prefix(model_info, "cache_creation_input_token_cost", usage, inclusive),
|
||||
)
|
||||
|
||||
|
||||
def _select_priced_tier(model_info: ModelInfo, usage: Usage) -> dict | None:
|
||||
|
|
@ -576,7 +656,9 @@ def _get_token_base_cost(
|
|||
# so that the threshold detection loop only processes standard keys. The
|
||||
# service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key.
|
||||
threshold_keys: Final = [
|
||||
k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES)
|
||||
k
|
||||
for k in model_info
|
||||
if k.startswith("input_cost_per_token_above_") and not k.endswith(_NON_STANDARD_THRESHOLD_SUFFIXES)
|
||||
]
|
||||
|
||||
# Only sort the threshold keys (typically 1-2 keys instead of 66+)
|
||||
|
|
@ -588,7 +670,7 @@ def _get_token_base_cost(
|
|||
# Handle both formats: _above_128k_tokens and _above_128_tokens
|
||||
threshold_str = key.split("_above_")[1].split("_tokens")[0]
|
||||
threshold = _parse_above_token_threshold(key)
|
||||
if usage.prompt_tokens > threshold or (threshold_is_inclusive and usage.prompt_tokens == threshold):
|
||||
if _prompt_exceeds_threshold(usage.prompt_tokens, threshold, threshold_is_inclusive):
|
||||
# Prefer a service_tier-specific above-threshold key when available,
|
||||
# e.g. input_cost_per_token_priority_above_200k_tokens for Gemini
|
||||
# ON_DEMAND_PRIORITY. Falls back to the standard key automatically
|
||||
|
|
|
|||
|
|
@ -251,6 +251,6 @@ def update_response_metadata(
|
|||
return
|
||||
|
||||
metadata: Final = ResponseMetadata(result)
|
||||
metadata.set_hidden_params(logging_obj, model, kwargs)
|
||||
metadata.set_timing_metrics(start_time, end_time, logging_obj, include_overhead)
|
||||
metadata.set_hidden_params(logging_obj, model, kwargs)
|
||||
metadata.apply()
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.classifier_logging import without_classifier_aud
|
|||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
)
|
||||
from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
redact_vertex_ai_metadata_from_litellm_params,
|
||||
redact_vertex_ai_metadata_from_logged_object,
|
||||
|
|
@ -245,6 +246,8 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
|
|||
if "audio" in choice["delta"]:
|
||||
choice["delta"]["audio"] = None
|
||||
_redact_tool_calls_dict(choice["delta"])
|
||||
elif choice.get("text") is not None:
|
||||
choice["text"] = redacted_str
|
||||
else:
|
||||
_redact_choice_content(choice)
|
||||
|
||||
|
|
@ -265,6 +268,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
|
|||
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
|
||||
model_call_details["prompt"] = ""
|
||||
model_call_details["input"] = ""
|
||||
model_call_details.pop(SERVED_OUTPUT_TEXTS_KEY, None)
|
||||
standard_logging_object: Final = model_call_details.get("standard_logging_object")
|
||||
if isinstance(standard_logging_object, Mapping):
|
||||
model_call_details["standard_logging_object"] = _redact_standard_logging_object(standard_logging_object)
|
||||
|
|
|
|||
171
litellm/litellm_core_utils/served_output_texts.py
Normal file
171
litellm/litellm_core_utils/served_output_texts.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
"""Assistant text the caller received, per choice, so the logging payload stores the response a
|
||||
post-call guardrail rewrote rather than the provider response the proxy assembled before it ran."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream
|
||||
|
||||
SERVED_OUTPUT_TEXTS_KEY: Final = "served_output_texts"
|
||||
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
|
||||
_JSON_LIST: Final = TypeAdapter(list[object])
|
||||
_TEXTS: Final = TypeAdapter(tuple[str | None, ...])
|
||||
|
||||
ServedTexts = tuple[str | None, ...]
|
||||
|
||||
|
||||
class _TextBlock(BaseModel):
|
||||
type: str
|
||||
text: str | None = None
|
||||
|
||||
|
||||
class _AnthropicMessage(BaseModel):
|
||||
type: Literal["message"]
|
||||
content: list[_TextBlock]
|
||||
|
||||
|
||||
class _ResponsesOutputItem(BaseModel):
|
||||
type: str
|
||||
content: list[_TextBlock] = []
|
||||
|
||||
|
||||
class _ResponsesResponse(BaseModel):
|
||||
object: Literal["response"]
|
||||
output: list[_ResponsesOutputItem]
|
||||
|
||||
|
||||
class _ChatChoices(BaseModel):
|
||||
choices: list[object]
|
||||
|
||||
|
||||
def _as_json_object(response: object) -> dict[str, object] | None:
|
||||
candidate: Final = response.model_dump() if isinstance(response, BaseModel) else response
|
||||
try:
|
||||
return _JSON_OBJECT.validate_python(candidate)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def _joined_block_texts(blocks: Sequence[_TextBlock], *, text_type: str) -> str | None:
|
||||
texts: Final = tuple(block.text for block in blocks if block.type == text_type and block.text is not None)
|
||||
return "".join(texts) if texts else None
|
||||
|
||||
|
||||
def _chat_texts(response: ModelResponse) -> ServedTexts | None:
|
||||
texts: Final = tuple(
|
||||
choice.message.content if isinstance(choice.message.content, str) else None for choice in response.choices
|
||||
)
|
||||
return texts if any(text is not None for text in texts) else None
|
||||
|
||||
|
||||
def _anthropic_message_text(response: dict[str, object]) -> str | None:
|
||||
try:
|
||||
message: Final = _AnthropicMessage.model_validate(response)
|
||||
except ValidationError:
|
||||
return None
|
||||
return _joined_block_texts(message.content, text_type="text")
|
||||
|
||||
|
||||
def _responses_api_text(response: dict[str, object]) -> str | None:
|
||||
try:
|
||||
parsed: Final = _ResponsesResponse.model_validate(response)
|
||||
except ValidationError:
|
||||
return None
|
||||
texts: Final = tuple(
|
||||
text
|
||||
for item in parsed.output
|
||||
if item.type == "message" and (text := _joined_block_texts(item.content, text_type="output_text")) is not None
|
||||
)
|
||||
return "".join(texts) if texts else None
|
||||
|
||||
|
||||
def _chat_dict_texts(response: dict[str, object]) -> ServedTexts | None:
|
||||
try:
|
||||
_ChatChoices.model_validate(response)
|
||||
return _chat_texts(ModelResponse(**response))
|
||||
except (ValidationError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def served_output_texts(response: object) -> ServedTexts | None:
|
||||
if isinstance(response, ModelResponse):
|
||||
return _chat_texts(response)
|
||||
mapping: Final = _as_json_object(response)
|
||||
if mapping is None:
|
||||
return None
|
||||
chat_texts: Final = _chat_dict_texts(mapping)
|
||||
if chat_texts is not None:
|
||||
return chat_texts
|
||||
anthropic_text: Final = _anthropic_message_text(mapping)
|
||||
text: Final = anthropic_text if anthropic_text is not None else _responses_api_text(mapping)
|
||||
return (text,) if text is not None else None
|
||||
|
||||
|
||||
def served_stream_output_texts(chunks: Sequence[object]) -> ServedTexts | None:
|
||||
if chunks and all(isinstance(chunk, ModelResponseStream) for chunk in chunks):
|
||||
return _chat_stream_texts(tuple(chunk for chunk in chunks if isinstance(chunk, ModelResponseStream)))
|
||||
from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_stream, is_anthropic_sse_stream
|
||||
|
||||
if not is_anthropic_sse_stream(chunks):
|
||||
return None
|
||||
assembled: Final = assemble_anthropic_sse_stream(chunks)
|
||||
return _chat_texts(assembled) if assembled is not None else None
|
||||
|
||||
|
||||
def _chat_stream_choice_text(chunks: Sequence[ModelResponseStream], index: int) -> str | None:
|
||||
contents: Final = tuple(
|
||||
content
|
||||
for chunk in chunks
|
||||
for choice in chunk.choices
|
||||
if choice.index == index and isinstance(content := choice.delta.content, str)
|
||||
)
|
||||
return "".join(contents) if contents else None
|
||||
|
||||
|
||||
def _chat_stream_texts(chunks: Sequence[ModelResponseStream]) -> ServedTexts | None:
|
||||
choice_count: Final = max((choice.index + 1 for chunk in chunks for choice in chunk.choices), default=0)
|
||||
texts: Final = tuple(_chat_stream_choice_text(chunks, index) for index in range(choice_count))
|
||||
return texts if any(text is not None for text in texts) else None
|
||||
|
||||
|
||||
def record_served_output_texts(model_call_details: dict[str, object], texts: ServedTexts | None) -> None:
|
||||
if texts is None:
|
||||
return
|
||||
model_call_details[SERVED_OUTPUT_TEXTS_KEY] = texts # rebind-ok: model_call_details is the shared kwargs bag
|
||||
|
||||
|
||||
def overlay_served_output_texts(
|
||||
response_obj: dict[str, object] | str | list[object] | None, served_texts: object
|
||||
) -> dict[str, object] | str | list[object] | None:
|
||||
if not isinstance(response_obj, dict):
|
||||
return response_obj
|
||||
logged: Final = _as_json_object(response_obj)
|
||||
if logged is None:
|
||||
return response_obj
|
||||
try:
|
||||
texts: Final = _TEXTS.validate_python(served_texts)
|
||||
choices: Final = _JSON_LIST.validate_python(logged.get("choices"))
|
||||
except ValidationError:
|
||||
return response_obj
|
||||
return {
|
||||
**logged,
|
||||
"choices": [
|
||||
_choice_with_text(choice, texts[index]) if index < len(texts) else choice
|
||||
for index, choice in enumerate(choices)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _choice_with_text(choice: object, text: str | None) -> object:
|
||||
choice_obj: Final = _as_json_object(choice)
|
||||
if choice_obj is None or text is None:
|
||||
return choice
|
||||
message: Final = _as_json_object(choice_obj.get("message"))
|
||||
if message is None or message.get("content") == text:
|
||||
return choice
|
||||
return {**choice_obj, "message": {**message, "content": text}}
|
||||
|
|
@ -10,7 +10,6 @@ import anyio
|
|||
import anyio.lowlevel
|
||||
import httpx
|
||||
import tiktoken
|
||||
from tokenizers import Tokenizer
|
||||
from typing_extensions import ParamSpec, TypeVar
|
||||
|
||||
import litellm
|
||||
|
|
@ -30,8 +29,10 @@ from litellm.constants import (
|
|||
TOKEN_COUNTER_MAX_EXACT_CHARS,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace, HuggingFaceTokenizer, OpenAIEncoding
|
||||
from litellm.litellm_core_utils.url_utils import safe_get
|
||||
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
|
||||
from litellm.rust_bridge.tokenizer import get_encoding
|
||||
from litellm.types.llms.anthropic import (
|
||||
AnthropicContentParamSource,
|
||||
AnthropicContentParamSourceFileId,
|
||||
|
|
@ -622,9 +623,11 @@ def _get_exact_count_function(
|
|||
if model is not None or custom_tokenizer is not None:
|
||||
tokenizer_json: Final = custom_tokenizer or _select_tokenizer(model)
|
||||
if tokenizer_json["type"] == "huggingface_tokenizer":
|
||||
tokenizer: Final[Tokenizer] = tokenizer_json["tokenizer"]
|
||||
tokenizer: Final[HuggingFace] = tokenizer_json["tokenizer"]
|
||||
|
||||
def count_tokens(text: str) -> int:
|
||||
if isinstance(tokenizer, HuggingFaceTokenizer):
|
||||
return tokenizer.count(text)
|
||||
return len(tokenizer.encode_batch_fast([text])[0])
|
||||
|
||||
return count_tokens
|
||||
|
|
@ -632,31 +635,43 @@ def _get_exact_count_function(
|
|||
encoding: Final = openai_tokenizer_encoding(model)
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return len(encoding.encode(text, disallowed_special=()))
|
||||
return _encoding_count(encoding, text)
|
||||
|
||||
return _get_tiktoken_count_function(encode_length)
|
||||
else:
|
||||
raise ValueError("Unsupported tokenizer type")
|
||||
else:
|
||||
default_encoding: Final = _get_default_encoding()
|
||||
|
||||
def encode_length(text: str) -> int:
|
||||
return len(_get_default_encoding().encode(text, disallowed_special=()))
|
||||
return _encoding_count(default_encoding, text)
|
||||
|
||||
return _get_tiktoken_count_function(encode_length)
|
||||
|
||||
|
||||
def openai_tokenizer_encoding(model: str) -> tiktoken.Encoding:
|
||||
"""The tiktoken encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
|
||||
def _encoding_count(encoding: Encoding, text: str) -> int:
|
||||
if isinstance(encoding, OpenAIEncoding):
|
||||
return encoding.count(text)
|
||||
return len(encoding.encode(text, disallowed_special=()))
|
||||
|
||||
|
||||
def openai_tokenizer_encoding(model: str) -> Encoding:
|
||||
"""The encoding `token_counter` uses for a model on the `openai_tokenizer` path."""
|
||||
return get_encoding(openai_tokenizer_encoding_name(model))
|
||||
|
||||
|
||||
def openai_tokenizer_encoding_name(model: str) -> str:
|
||||
"""The tiktoken encoding name for `model`, without loading the encoding."""
|
||||
from litellm.utils import print_verbose
|
||||
|
||||
model_to_use: Final = _fix_model_name(model)
|
||||
if "gpt-4o" in model_to_use:
|
||||
return tiktoken.get_encoding("o200k_base")
|
||||
return "o200k_base"
|
||||
try:
|
||||
return tiktoken.encoding_for_model(model_to_use)
|
||||
return tiktoken.encoding_name_for_model(model_to_use)
|
||||
except KeyError:
|
||||
print_verbose("Warning: model not found. Using cl100k_base encoding.")
|
||||
return tiktoken.get_encoding("cl100k_base")
|
||||
return "cl100k_base"
|
||||
|
||||
|
||||
def uses_legacy_message_accounting(model: str) -> bool:
|
||||
|
|
|
|||
402
litellm/litellm_core_utils/tokenizer.py
Normal file
402
litellm/litellm_core_utils/tokenizer.py
Normal file
|
|
@ -0,0 +1,402 @@
|
|||
"""Python faces of the Rust text codecs.
|
||||
|
||||
``OpenAIEncoding`` mirrors ``tiktoken.Encoding`` and ``HuggingFaceTokenizer`` mirrors
|
||||
``tokenizers.Tokenizer``, so a caller holding ``litellm.encoding`` or the object returned by
|
||||
``litellm.create_tokenizer`` sees the same read-only surface whichever backend the Rust catalog
|
||||
selected. Both wrappers are immutable: ``tokenizers`` mutators (``enable_padding``,
|
||||
``enable_truncation``, ``add_tokens``) stay on the Python tokenizer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Collection, Mapping, Sequence, Set
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, runtime_checkable
|
||||
|
||||
import tiktoken
|
||||
from tokenizers import AddedToken
|
||||
from tokenizers import Tokenizer as PythonHuggingFaceTokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from litellm.rust_bridge._native import HuggingFaceEncoding
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
SpecialTokens: TypeAlias = Literal["all"] | Collection[str]
|
||||
AllowedSpecial: TypeAlias = Literal["all"] | Set[str]
|
||||
HuggingFaceInput: TypeAlias = str | list[str] | tuple[str, ...]
|
||||
HuggingFaceBatchInput: TypeAlias = HuggingFaceInput | tuple[HuggingFaceInput, HuggingFaceInput] | list[HuggingFaceInput]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class OpenAIEncoding:
|
||||
"""``tiktoken.Encoding`` over the Rust tiktoken codec."""
|
||||
|
||||
_native: NativeTokenizer
|
||||
_special_tokens: Mapping[str, int]
|
||||
|
||||
@staticmethod
|
||||
def wrap(native: NativeTokenizer) -> OpenAIEncoding:
|
||||
return OpenAIEncoding(native, MappingProxyType(native.special_tokens()))
|
||||
|
||||
@staticmethod
|
||||
def from_tiktoken(encoding: str) -> OpenAIEncoding:
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
return OpenAIEncoding.wrap(NativeTokenizer.from_tiktoken(encoding))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Encoding {self.name!r}>"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._native.name
|
||||
|
||||
@property
|
||||
def max_token_value(self) -> int:
|
||||
return self._native.max_token_value()
|
||||
|
||||
@property
|
||||
def n_vocab(self) -> int:
|
||||
"""For backwards compatibility. Prefer to use `enc.max_token_value + 1`."""
|
||||
return self.max_token_value + 1
|
||||
|
||||
@property
|
||||
def eot_token(self) -> int:
|
||||
return self._special_tokens["<|endoftext|>"]
|
||||
|
||||
@property
|
||||
def special_tokens_set(self) -> set[str]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return set(self._special_tokens)
|
||||
|
||||
def is_special_token(self, token: int) -> bool:
|
||||
return self._native.is_special_token(token)
|
||||
|
||||
# ---- encoding -------------------------------------------------------------------------
|
||||
|
||||
def encode_ordinary(self, text: str) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.encode(text)
|
||||
|
||||
def encode(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> list[int]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
|
||||
if not allowed:
|
||||
return self.encode_ordinary(text)
|
||||
return self._native.encode_special(text, tuple(allowed))
|
||||
|
||||
def encode_to_numpy(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> npt.NDArray[np.uint32]:
|
||||
import numpy
|
||||
|
||||
return numpy.asarray(
|
||||
self.encode(text, allowed_special=allowed_special, disallowed_special=disallowed_special),
|
||||
dtype=numpy.uint32,
|
||||
)
|
||||
|
||||
def encode_ordinary_batch(
|
||||
self, text: Sequence[str], *, num_threads: int = 8
|
||||
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(self.encode_ordinary, text)
|
||||
)
|
||||
|
||||
def encode_batch(
|
||||
self,
|
||||
text: Sequence[str],
|
||||
*,
|
||||
num_threads: int = 8,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> list[list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
encode: Final = partial(self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special)
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(encode, text)
|
||||
)
|
||||
|
||||
def encode_with_unstable(
|
||||
self,
|
||||
text: str,
|
||||
*,
|
||||
allowed_special: AllowedSpecial = frozenset(),
|
||||
disallowed_special: SpecialTokens = "all",
|
||||
) -> tuple[list[int], list[list[int]]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
"""The stable tokens of `text` and every completion its unstable tail could become.
|
||||
|
||||
Completions come back sorted; tiktoken returns them in hash order."""
|
||||
allowed: Final = self._allowed(text, allowed_special, disallowed_special)
|
||||
return self._native.encode_with_unstable(text, tuple(allowed))
|
||||
|
||||
def encode_single_token(self, text_or_bytes: str | bytes) -> int:
|
||||
"""The token of one whole piece, special tokens included. Raises `KeyError` otherwise."""
|
||||
piece: Final = text_or_bytes.encode("utf-8") if isinstance(text_or_bytes, str) else text_or_bytes
|
||||
return self._native.encode_single_token(piece)
|
||||
|
||||
def count(self, text: str, fast: bool = False) -> int:
|
||||
"""Count ordinary text; `fast` accelerates supported encodings and otherwise counts normally."""
|
||||
return self._native.count(text, fast)
|
||||
|
||||
# ---- decoding -------------------------------------------------------------------------
|
||||
|
||||
def decode_bytes(self, tokens: Sequence[int]) -> bytes:
|
||||
return self._native.decode_bytes(tokens)
|
||||
|
||||
def decode(self, tokens: Sequence[int], errors: str = "replace") -> str:
|
||||
return self.decode_bytes(tokens).decode("utf-8", errors=errors)
|
||||
|
||||
def decode_single_token_bytes(self, token: int) -> bytes:
|
||||
return self.decode_bytes((token,))
|
||||
|
||||
def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return [ # mutable-ok: [LIT002] SDK returns a list
|
||||
self.decode_single_token_bytes(token) for token in tokens
|
||||
]
|
||||
|
||||
def decode_with_offsets(
|
||||
self, tokens: Sequence[int]
|
||||
) -> tuple[str, list[int]]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
"""The decoded text and, per token, the index of the first character holding its bytes.
|
||||
|
||||
Like tiktoken, raises `UnicodeDecodeError` when the tokens do not decode to valid UTF-8."""
|
||||
token_bytes: Final = self.decode_tokens_bytes(tokens)
|
||||
text_len = 0
|
||||
offsets: Final[list[int]] = [] # mutable-ok: [LIT001] local accumulator
|
||||
for token in token_bytes:
|
||||
offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0)))
|
||||
text_len += sum(1 for c in token if not 0x80 <= c < 0xC0)
|
||||
return b"".join(token_bytes).decode("utf-8", errors="strict"), offsets
|
||||
|
||||
def decode_batch(
|
||||
self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8
|
||||
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(partial(self.decode, errors=errors), batch)
|
||||
)
|
||||
|
||||
def decode_bytes_batch(
|
||||
self, batch: Sequence[Sequence[int]], *, num_threads: int = 8
|
||||
) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
with ThreadPoolExecutor(num_threads) as executor:
|
||||
return list( # mutable-ok: [LIT002] SDK returns a list
|
||||
executor.map(self.decode_bytes, batch)
|
||||
)
|
||||
|
||||
def token_byte_values(self) -> list[bytes]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.token_byte_values()
|
||||
|
||||
def __reduce__(self) -> tuple[Callable[[str], OpenAIEncoding], tuple[str]]:
|
||||
return (OpenAIEncoding.from_tiktoken, (self.name,))
|
||||
|
||||
# ---- private --------------------------------------------------------------------------
|
||||
|
||||
def _allowed(self, text: str, allowed_special: AllowedSpecial, disallowed_special: SpecialTokens) -> frozenset[str]:
|
||||
"""tiktoken's special-token policy: which specials `text` may encode, after rejecting
|
||||
any it must not contain."""
|
||||
allowed: Final = frozenset(self._special_tokens) if allowed_special == "all" else frozenset(allowed_special)
|
||||
disallowed: Final = (
|
||||
frozenset(self._special_tokens) - allowed if disallowed_special == "all" else frozenset(disallowed_special)
|
||||
)
|
||||
for token in disallowed:
|
||||
if token in text:
|
||||
raise ValueError(
|
||||
f"Encountered text corresponding to disallowed special token {token!r}.\n"
|
||||
"If you want this text to be encoded as a special token, "
|
||||
f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n"
|
||||
"If you want this text to be encoded as normal text, disable the check for this token "
|
||||
f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n"
|
||||
"To disable this check for all special tokens, pass `disallowed_special=()`.\n"
|
||||
)
|
||||
return allowed
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class HuggingFaceTokenizer:
|
||||
"""The read-only ``tokenizers.Tokenizer`` surface over the Rust Hugging Face codec."""
|
||||
|
||||
_native: NativeTokenizer
|
||||
|
||||
@staticmethod
|
||||
def from_str(json: str) -> HuggingFaceTokenizer:
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
return HuggingFaceTokenizer(NativeTokenizer.from_json(json))
|
||||
|
||||
from_json = from_str
|
||||
|
||||
@staticmethod
|
||||
def from_buffer(buffer: bytes) -> HuggingFaceTokenizer:
|
||||
return HuggingFaceTokenizer.from_str(buffer.decode("utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def from_file(path: str) -> HuggingFaceTokenizer:
|
||||
return HuggingFaceTokenizer.from_str(Path(path).read_text(encoding="utf-8"))
|
||||
|
||||
@staticmethod
|
||||
def from_pretrained(identifier: str, revision: str = "main", token: str | None = None) -> HuggingFaceTokenizer:
|
||||
from litellm.rust_bridge._native import Tokenizer as NativeTokenizer
|
||||
|
||||
return HuggingFaceTokenizer(NativeTokenizer.from_pretrained(identifier, revision=revision, token=token))
|
||||
|
||||
def to_str(self, pretty: bool = False) -> str:
|
||||
return self._native.to_json(pretty)
|
||||
|
||||
def save(self, path: str, pretty: bool = True) -> None:
|
||||
Path(path).write_text(self.to_str(pretty), encoding="utf-8")
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._native.name
|
||||
|
||||
# ---- vocabulary -----------------------------------------------------------------------
|
||||
|
||||
def token_to_id(self, token: str) -> int | None:
|
||||
return self._native.token_to_id(token)
|
||||
|
||||
def id_to_token(self, id: int) -> str | None:
|
||||
return self._native.id_to_token(id)
|
||||
|
||||
def get_vocab(
|
||||
self, with_added_tokens: bool = True
|
||||
) -> dict[str, int]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.get_vocab(with_added_tokens)
|
||||
|
||||
def get_vocab_size(self, with_added_tokens: bool = True) -> int:
|
||||
return self._native.get_vocab_size(with_added_tokens)
|
||||
|
||||
def get_added_tokens_decoder(self) -> dict[int, AddedToken]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return { # mutable-ok: [LIT002] SDK returns a dict
|
||||
token_id: AddedToken(
|
||||
content, single_word=single_word, lstrip=lstrip, rstrip=rstrip, normalized=normalized, special=special
|
||||
)
|
||||
for token_id, (
|
||||
content,
|
||||
single_word,
|
||||
lstrip,
|
||||
rstrip,
|
||||
normalized,
|
||||
special,
|
||||
) in self._native.added_tokens_decoder()
|
||||
}
|
||||
|
||||
def num_special_tokens_to_add(self, is_pair: bool) -> int:
|
||||
return self._native.num_special_tokens_to_add(is_pair)
|
||||
|
||||
@property
|
||||
def padding(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.padding()
|
||||
|
||||
@property
|
||||
def truncation(self) -> dict[str, object] | None: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._native.truncation()
|
||||
|
||||
@property
|
||||
def encode_special_tokens(self) -> bool:
|
||||
return self._native.encode_special_tokens()
|
||||
|
||||
# ---- encoding and decoding ------------------------------------------------------------
|
||||
|
||||
def encode(
|
||||
self,
|
||||
sequence: HuggingFaceInput,
|
||||
pair: HuggingFaceInput | None = None,
|
||||
is_pretokenized: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
) -> HuggingFaceEncoding:
|
||||
return self._native.encode_huggingface(sequence, pair, is_pretokenized, add_special_tokens)
|
||||
|
||||
def encode_batch(
|
||||
self,
|
||||
input: Sequence[HuggingFaceBatchInput],
|
||||
is_pretokenized: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=False)
|
||||
|
||||
def encode_batch_fast(
|
||||
self,
|
||||
input: Sequence[HuggingFaceBatchInput],
|
||||
is_pretokenized: bool = False,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return self._encode_batch(input, is_pretokenized, add_special_tokens, fast=True)
|
||||
|
||||
def _encode_batch(
|
||||
self, input: Sequence[HuggingFaceBatchInput], is_pretokenized: bool, add_special_tokens: bool, fast: bool
|
||||
) -> list[HuggingFaceEncoding]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
sequences: Final = tuple(_batch_input(item, is_pretokenized) for item in input)
|
||||
return self._native.encode_batch_huggingface(sequences, is_pretokenized, add_special_tokens, fast)
|
||||
|
||||
def count(self, text: str, fast: bool = False) -> int:
|
||||
"""Count with this tokenizer's configuration; `fast` uses acceleration where supported."""
|
||||
return self._native.count(text, fast)
|
||||
|
||||
def decode(self, ids: Sequence[int], skip_special_tokens: bool = True) -> str:
|
||||
return self._native.decode(ids, skip_special_tokens=skip_special_tokens)
|
||||
|
||||
def decode_batch(
|
||||
self, sequences: Sequence[Sequence[int]], skip_special_tokens: bool = True
|
||||
) -> list[str]: # mutable-ok: [LIT001, LIT002] SDK return type
|
||||
return [ # mutable-ok: [LIT002] SDK returns a list
|
||||
self.decode(ids, skip_special_tokens=skip_special_tokens) for ids in sequences
|
||||
]
|
||||
|
||||
def __reduce__(self) -> tuple[Callable[[str], HuggingFaceTokenizer], tuple[str]]:
|
||||
return (HuggingFaceTokenizer.from_str, (self.to_str(),))
|
||||
|
||||
|
||||
def _batch_input(
|
||||
item: HuggingFaceBatchInput, is_pretokenized: bool
|
||||
) -> tuple[HuggingFaceInput, HuggingFaceInput | None]:
|
||||
if isinstance(item, str):
|
||||
return (item, None)
|
||||
if is_pretokenized and all(isinstance(word, str) for word in item):
|
||||
return (tuple(word for word in item if isinstance(word, str)), None)
|
||||
if len(item) != 2:
|
||||
raise TypeError("batch input must be a sequence or a pair of sequences")
|
||||
return (item[0], item[1])
|
||||
|
||||
|
||||
Encoding: TypeAlias = tiktoken.Encoding | OpenAIEncoding
|
||||
HuggingFace: TypeAlias = PythonHuggingFaceTokenizer | HuggingFaceTokenizer
|
||||
Tokenizer: TypeAlias = Encoding | HuggingFace
|
||||
|
||||
|
||||
class _AddedToken(Protocol):
|
||||
@property
|
||||
def special(self) -> bool: ...
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class _AddedTokenDecoder(Protocol):
|
||||
def get_added_tokens_decoder(self) -> Mapping[int, _AddedToken]: ...
|
||||
|
||||
|
||||
def strip_special_tokens(tokenizer: object, tokens: Sequence[int]) -> Sequence[int]:
|
||||
"""Drop the special added tokens before a Python `tokenizers` decode; the Rust codec's
|
||||
`decode(skip_special_tokens=True)` already does this itself."""
|
||||
if isinstance(tokenizer, HuggingFaceTokenizer) or not isinstance(tokenizer, _AddedTokenDecoder):
|
||||
return tokens
|
||||
try:
|
||||
added: Final = tokenizer.get_added_tokens_decoder()
|
||||
except Exception: # noqa: BLE001 # optional metadata failures historically fall back to decoding
|
||||
return tokens
|
||||
special_ids: Final = frozenset(token_id for token_id, token in added.items() if token.special)
|
||||
return tuple(token for token in tokens if token not in special_ids)
|
||||
|
|
@ -23,9 +23,8 @@ from ..common_utils import (
|
|||
from .streaming_iterator import A2AModelResponseIterator
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
_REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = (
|
||||
|
|
@ -292,7 +291,7 @@ class A2AConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -14,9 +14,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import ImageObject, ImageResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -171,7 +170,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -16,9 +16,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -68,7 +67,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from litellm.types.utils import ModelResponse
|
|||
from ...openai_like.chat.transformation import OpenAILikeChatConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AmazonNovaChatConfig(OpenAILikeChatConfig):
|
||||
|
|
@ -86,7 +86,7 @@ class AmazonNovaChatConfig(OpenAILikeChatConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -14,9 +14,8 @@ from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
|
|||
from litellm.types.utils import LiteLLMBatch, LlmProviders, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -290,7 +289,7 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -96,13 +96,13 @@ from ..common_utils import (
|
|||
AnthropicModelInfo,
|
||||
eager_input_streaming_flag,
|
||||
process_anthropic_headers,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -1771,7 +1771,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
return tools
|
||||
|
||||
def _ensure_beta_header(self, headers: dict, beta_value: str) -> None:
|
||||
def _ensure_beta_header(self, headers: dict[str, str], beta_value: str) -> None:
|
||||
"""
|
||||
Ensure a beta header value is present in the anthropic-beta header.
|
||||
Merges with existing values instead of overriding them.
|
||||
|
|
@ -1780,13 +1780,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers: Dictionary of headers to update
|
||||
beta_value: The beta header value to add
|
||||
"""
|
||||
existing_beta: Final = headers.get("anthropic-beta")
|
||||
if existing_beta is None:
|
||||
headers["anthropic-beta"] = beta_value
|
||||
return
|
||||
existing_values: Final = [beta.strip() for beta in existing_beta.split(",")]
|
||||
if beta_value not in existing_values:
|
||||
headers["anthropic-beta"] = f"{existing_beta}, {beta_value}"
|
||||
existing_values: Final = tuple(
|
||||
beta.strip()
|
||||
for key, value in headers.items()
|
||||
if key.lower() == "anthropic-beta"
|
||||
for beta in value.split(",")
|
||||
if beta.strip()
|
||||
)
|
||||
for key in tuple(headers):
|
||||
if key.lower() == "anthropic-beta":
|
||||
headers.pop(key)
|
||||
headers["anthropic-beta"] = ", ".join(dict.fromkeys((*existing_values, beta_value)))
|
||||
|
||||
def _ensure_context_management_beta_header(self, headers: dict, context_management: object) -> None:
|
||||
"""
|
||||
|
|
@ -1824,7 +1828,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
|
||||
)
|
||||
|
||||
def update_headers_with_optional_anthropic_beta(self, headers: dict, optional_params: dict) -> dict:
|
||||
def update_headers_with_optional_anthropic_beta(
|
||||
self, headers: dict, optional_params: dict, messages: Sequence[object] = ()
|
||||
) -> dict:
|
||||
"""Update headers with optional anthropic beta."""
|
||||
|
||||
# Skip adding beta headers for Vertex requests
|
||||
|
|
@ -1833,6 +1839,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
if is_vertex_request:
|
||||
return headers
|
||||
|
||||
if requires_native_compaction_beta(self._resolved_provider, optional_params, messages):
|
||||
self._ensure_beta_header(headers, ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
_tools: Final = optional_params.get("tools", [])
|
||||
for tool in _tools:
|
||||
if tool.get("type", None) and tool.get("type").startswith(ANTHROPIC_HOSTED_TOOLS.WEB_FETCH.value):
|
||||
|
|
@ -1929,8 +1938,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
custom_llm_provider=self._resolved_provider,
|
||||
)
|
||||
|
||||
headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
|
||||
|
||||
# === Tool-name sanitization (single chokepoint) ===
|
||||
# Anthropic enforces ^[a-zA-Z0-9_-]{1,128}$ on every tool name. We
|
||||
# sanitize *here* -- not in map_openai_params -- because:
|
||||
|
|
@ -1977,6 +1984,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
message=f"{e}\nReceived Messages={messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
headers=headers, optional_params=optional_params, messages=anthropic_messages
|
||||
)
|
||||
|
||||
## Auto-strip advisor blocks from history if advisor tool is absent.
|
||||
## Prevents Anthropic 400: advisor_tool_result in history requires advisor tool.
|
||||
_all_tools: Final = optional_params.get("tools") or []
|
||||
|
|
@ -2688,7 +2699,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -79,6 +79,27 @@ _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
|
|||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
||||
|
||||
def requires_native_compaction_beta(
|
||||
custom_llm_provider: str,
|
||||
optional_params: Mapping[str, object],
|
||||
messages: Sequence[object],
|
||||
) -> bool:
|
||||
return custom_llm_provider == "anthropic" and (
|
||||
optional_params.get("compaction") is not None
|
||||
or any(
|
||||
isinstance(block, Mapping)
|
||||
and block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and bool(block.get("signature"))
|
||||
for message in messages
|
||||
if isinstance(message, Mapping)
|
||||
for content in (message.get("content"),)
|
||||
if isinstance(content, (list, tuple))
|
||||
for block in content
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def supports_anthropic_cache_control(model: str, custom_llm_provider: str | None) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.utils import supports_prompt_caching
|
||||
|
|
|
|||
71
litellm/llms/anthropic/compaction.py
Normal file
71
litellm/llms/anthropic/compaction.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.llms.compaction import CompactionProtocol
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES, AnthropicCompaction
|
||||
|
||||
_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_OBJECTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_HEADERS: Final = TypeAdapter(dict[str, str])
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_CONFLICTS: Final = ("context_management", "response_format", "stop", "stop_sequences", "tool_choice")
|
||||
|
||||
|
||||
def supports_native_compaction(params: Mapping[str, object]) -> bool:
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
if params.get("custom_llm_provider") not in (None, "anthropic", "openai"):
|
||||
return False
|
||||
model: Final = str(params.get("model", "")).removeprefix("openai/").removeprefix("anthropic/")
|
||||
try:
|
||||
return get_model_info(model=model, custom_llm_provider="anthropic").get("supports_anthropic_compaction") is True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def compatible_defaults(payload: Mapping[str, object]) -> bool:
|
||||
return all(payload.get(key) is None for key in _CONFLICTS)
|
||||
|
||||
|
||||
def request_kwargs() -> Mapping[str, object]:
|
||||
operation: Final[AnthropicCompaction] = {"type": "summarize"}
|
||||
return MappingProxyType(
|
||||
{
|
||||
"compaction": operation,
|
||||
"extra_headers": _HEADERS.validate_python(
|
||||
MappingProxyType({"anthropic-beta": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value})
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _native_blocks(protocol: CompactionProtocol, response: Mapping[str, object]) -> tuple[Mapping[str, object], ...]:
|
||||
if protocol == "messages":
|
||||
return (
|
||||
_OBJECTS.validate_python(response.get("content", ())) if response.get("stop_reason") == "compaction" else ()
|
||||
)
|
||||
choices: Final = _OBJECTS.validate_python(response.get("choices", ()))
|
||||
choice: Final = choices[0] if len(choices) == 1 else _EMPTY
|
||||
message: Final = _MAPPING.validate_python(choice.get("message", _EMPTY))
|
||||
fields: Final = _MAPPING.validate_python(message.get("provider_specific_fields") or _EMPTY)
|
||||
return _OBJECTS.validate_python(fields.get("compaction_blocks", ()))
|
||||
|
||||
|
||||
def extract_summary(protocol: CompactionProtocol, response: Mapping[str, object]) -> str | None:
|
||||
blocks: Final = _native_blocks(protocol, response)
|
||||
block: Final = blocks[0] if len(blocks) == 1 else _EMPTY
|
||||
content: Final = block.get("content")
|
||||
return (
|
||||
content
|
||||
if block.get("type") == "compaction"
|
||||
and isinstance(block.get("signature"), str)
|
||||
and block.get("signature")
|
||||
and isinstance(content, str)
|
||||
and content.strip()
|
||||
else None
|
||||
)
|
||||
|
|
@ -33,7 +33,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AnthropicTextError(BaseLLMException):
|
||||
|
|
@ -185,7 +185,7 @@ class AnthropicTextConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -2,8 +2,11 @@ import copy
|
|||
import hashlib
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
|
||||
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.experimental_pass_through.utils import (
|
||||
is_reasoning_auto_summary_enabled,
|
||||
|
|
@ -16,6 +19,7 @@ OPENAI_MAX_TOOL_NAME_LENGTH: Final = 64
|
|||
TOOL_NAME_HASH_LENGTH: Final = 8
|
||||
TOOL_NAME_PREFIX_LENGTH: Final = OPENAI_MAX_TOOL_NAME_LENGTH - TOOL_NAME_HASH_LENGTH - 1 # 55
|
||||
PROVIDERS_PROXYING_AN_UNKNOWN_BACKEND: Final = frozenset({"litellm_proxy"})
|
||||
_COMPACTION_BLOCK: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _optional_attr(source: object, name: str) -> object:
|
||||
|
|
@ -36,6 +40,20 @@ def _thought_signature(provider_specific_fields: object) -> str | None:
|
|||
return signature if isinstance(signature, str) else None
|
||||
|
||||
|
||||
def _compaction_blocks(provider_specific_fields: object) -> tuple[Mapping[str, object], ...]:
|
||||
fields: Final = _as_string_mapping(provider_specific_fields)
|
||||
raw_blocks: Final = fields.get("compaction_blocks") if fields is not None else None
|
||||
return (
|
||||
tuple(
|
||||
block
|
||||
for raw_block in raw_blocks
|
||||
if (block := _as_string_mapping(raw_block)) is not None and block.get("type") == "compaction"
|
||||
)
|
||||
if isinstance(raw_blocks, (list, tuple))
|
||||
else ()
|
||||
)
|
||||
|
||||
|
||||
_ANTHROPIC_TOOL_SCHEMA_KEYS: Final = frozenset(
|
||||
{"name", "type", "input_schema", "description", "cache_control", "strict"}
|
||||
)
|
||||
|
|
@ -1330,7 +1348,11 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
tool_name_mapping: dict[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
new_content: Final[list[dict[str, Any]]] = []
|
||||
for choice in choices:
|
||||
for choice, compaction_blocks in (
|
||||
(choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields")))
|
||||
for choice in choices
|
||||
):
|
||||
new_content.extend(_COMPACTION_BLOCK.validate_python(block) for block in compaction_blocks)
|
||||
# Handle thinking blocks first
|
||||
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
|
||||
for thinking_block in choice.message.thinking_blocks:
|
||||
|
|
@ -1365,7 +1387,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
)
|
||||
|
||||
# Handle text content
|
||||
if choice.message.content is not None:
|
||||
if choice.message.content is not None and (choice.message.content != "" or not compaction_blocks):
|
||||
new_content.append(
|
||||
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
|
||||
)
|
||||
|
|
@ -1545,21 +1567,35 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
openai_finish_reason=openai_finish_reason
|
||||
)
|
||||
anthropic_finish_reason: Final = (
|
||||
"refusal"
|
||||
"compaction"
|
||||
if len(anthropic_content) == 1 and anthropic_content[0].get("type") == "compaction"
|
||||
else "refusal"
|
||||
if refusal_text is not None and translated_finish_reason != "max_tokens"
|
||||
else translated_finish_reason
|
||||
)
|
||||
# extract usage
|
||||
usage: Final[Usage] = getattr(response, "usage")
|
||||
anthropic_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
|
||||
|
||||
if polyfill_result is not None and polyfill_result.iterations_usage is not None:
|
||||
message_iteration: Final[UsageIteration] = {
|
||||
"type": "message",
|
||||
"input_tokens": anthropic_usage["input_tokens"],
|
||||
"output_tokens": usage.completion_tokens or 0,
|
||||
}
|
||||
anthropic_usage["iterations"] = list(polyfill_result.iterations_usage) + [message_iteration]
|
||||
message_usage: Final = self._translate_openai_usage_to_anthropic_usage(usage)
|
||||
polyfill_iterations: Final = polyfill_result.iterations_usage if polyfill_result is not None else None
|
||||
anthropic_usage: Final[AnthropicUsage] = (
|
||||
TypeAdapter(AnthropicUsage).validate_python(
|
||||
MappingProxyType(
|
||||
{
|
||||
**message_usage,
|
||||
"iterations": (
|
||||
*polyfill_iterations,
|
||||
UsageIteration(
|
||||
type="message",
|
||||
input_tokens=message_usage.get("input_tokens", 0),
|
||||
output_tokens=usage.completion_tokens or 0,
|
||||
),
|
||||
),
|
||||
}
|
||||
)
|
||||
)
|
||||
if polyfill_iterations is not None
|
||||
else message_usage
|
||||
)
|
||||
|
||||
translated_obj: Final = AnthropicMessagesResponse(
|
||||
id=response.id,
|
||||
|
|
|
|||
|
|
@ -580,7 +580,9 @@ def anthropic_messages_handler(
|
|||
)
|
||||
if anthropic_messages_provider_config is None:
|
||||
# Route to Responses API for OpenAI / Azure, chat/completions for everything else.
|
||||
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
|
||||
if kwargs.get("compaction") is None and _should_route_to_responses_api(
|
||||
custom_llm_provider, original_model, model
|
||||
):
|
||||
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
|
||||
max_tokens=max_tokens,
|
||||
messages=messages,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from ...common_utils import (
|
|||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
optionally_handle_anthropic_oauth,
|
||||
requires_native_compaction_beta,
|
||||
strip_advisor_blocks_from_messages,
|
||||
strip_encrypted_reasoning_blocks_from_anthropic_messages,
|
||||
)
|
||||
|
|
@ -74,6 +75,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"tool_choice",
|
||||
"thinking",
|
||||
"context_management",
|
||||
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
|
||||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
|
|
@ -637,6 +639,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
)
|
||||
beta_values.update(existing_beta)
|
||||
|
||||
if requires_native_compaction_beta(custom_llm_provider, optional_params, messages):
|
||||
beta_values.add(ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value)
|
||||
|
||||
# Check for context management
|
||||
context_management_param: Final = optional_params.get("context_management")
|
||||
if context_management_param is not None:
|
||||
|
|
|
|||
|
|
@ -29,9 +29,8 @@ from ...base_llm.chat.transformation import BaseConfig
|
|||
from ..common_utils import AzureOpenAIError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LoggingClass = LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -304,7 +303,7 @@ class AzureOpenAIConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -143,7 +143,7 @@ class AzurePassthroughConfig(BasePassthroughConfig):
|
|||
) -> dict:
|
||||
return BaseAzureLLM._base_validate_azure_environment(
|
||||
headers=headers,
|
||||
litellm_params=GenericLiteLLMParams(**{**litellm_params, "api_key": api_key}),
|
||||
litellm_params=GenericLiteLLMParams.model_validate({**litellm_params, "api_key": api_key}),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -34,9 +34,8 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
|
|
@ -297,7 +296,7 @@ class AzureAIAgentsConfig(BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from litellm.types.llms.openai import AllMessageValues
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AzureModelRouterConfig(AzureAIStudioConfig):
|
||||
|
|
@ -59,7 +59,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ from litellm.types.utils import ModelResponse, ProviderField
|
|||
from litellm.utils import _add_path_to_api_base, supports_tool_choice
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AzureFoundryErrorStrings(str, enum.Enum):
|
||||
|
|
@ -305,7 +305,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -12,9 +12,10 @@ from litellm.types.utils import ImageResponse
|
|||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
|
||||
|
|
@ -245,7 +246,7 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
|
|||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
|
|
|
|||
|
|
@ -12,9 +12,8 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -121,7 +120,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ from collections.abc import AsyncIterator, Iterator
|
|||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
from pydantic import BaseModel
|
||||
|
||||
from litellm import LiteLLMLoggingObj, ModelResponse
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
|
||||
|
|
@ -39,7 +39,7 @@ class CompletionTransformationBridge(ABC):
|
|||
messages: list["AllMessageValues"],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> "ModelResponse":
|
||||
|
|
|
|||
|
|
@ -21,9 +21,8 @@ from litellm.types.llms.openai import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
from ..base_utils import (
|
||||
|
|
@ -344,7 +343,7 @@ class BaseConfig(ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> "ModelResponse":
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUser
|
|||
from litellm.types.utils import ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -68,7 +67,7 @@ class BaseTextCompletionConfig(BaseConfig, ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
|
|
@ -8,9 +8,8 @@ from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
|
|||
from litellm.types.utils import EmbeddingResponse, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import tiktoken
|
||||
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
|
|
@ -80,7 +79,7 @@ class BaseEmbeddingConfig(BaseConfig, ABC):
|
|||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "tiktoken.Encoding | None",
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ModelResponse:
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue