From 4432805bf72f95a63307f87a7d0cf1a6b4538696 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 23 Oct 2026 12:33:59 -0700 Subject: [PATCH] refactor(rust-error): keep causes as sources across http, auth, secrets, cache and utility crates Messages describe their own layer and every cause is a typed source, predicates match exhaustively, and boundaries render with litellm_error::Report. Media downloads no longer count as dispatched, Error::upstream bounds the provider body itself, and a forwarded header SigV4 cannot sign is the caller's fault. Co-Authored-By: Claude Opus 5.5 --- .../references/messages-and-causes.md | 2 +- .../skills/rust-error/references/testing.md | 4 +- litellm-rust/Cargo.lock | 36 ++ litellm-rust/crates/auth-aws/Cargo.toml | 1 + litellm-rust/crates/auth-aws/src/aws.rs | 92 +++- litellm-rust/crates/auth-aws/src/error.rs | 101 ++-- litellm-rust/crates/auth-aws/src/lib.rs | 2 +- litellm-rust/crates/auth-aws/src/signer.rs | 28 +- litellm-rust/crates/auth-azure/Cargo.toml | 1 + litellm-rust/crates/auth-azure/src/native.rs | 43 +- litellm-rust/crates/auth-azure/src/resolve.rs | 111 +++-- litellm-rust/crates/auth-gcp/Cargo.toml | 1 + litellm-rust/crates/auth-gcp/src/auth.rs | 16 +- litellm-rust/crates/auth-gcp/src/sdk.rs | 6 +- litellm-rust/crates/auth-types/Cargo.toml | 1 + .../crates/auth-types/src/credential.rs | 26 +- litellm-rust/crates/auth-types/src/error.rs | 287 ++++++++--- litellm-rust/crates/auth-types/src/http.rs | 12 +- litellm-rust/crates/auth-types/src/lib.rs | 4 +- .../crates/auth-types/src/params/aws.rs | 49 +- .../crates/auth-types/src/params/mod.rs | 7 + .../crates/auth-types/src/params/vertex.rs | 5 +- litellm-rust/crates/auth-types/src/policy.rs | 35 +- litellm-rust/crates/auth-types/tests/error.rs | 81 ---- .../crates/auth-types/tests/params.rs | 3 +- .../crates/cache-azure-blob/Cargo.toml | 1 + .../crates/cache-azure-blob/src/cache.rs | 30 +- .../crates/cache-azure-blob/src/credential.rs | 15 +- .../crates/cache-azure-blob/src/transport.rs | 8 +- .../crates/cache-azure-blob/tests/cache.rs | 12 +- litellm-rust/crates/cache-disk/Cargo.toml | 1 + litellm-rust/crates/cache-disk/src/cache.rs | 8 +- .../crates/cache-disk/src/python/value.rs | 18 +- litellm-rust/crates/cache-disk/src/sqlite.rs | 168 ++++--- .../crates/cache-disk/tests/python_compat.rs | 2 +- litellm-rust/crates/cache-gcs/Cargo.toml | 2 + litellm-rust/crates/cache-gcs/src/cache.rs | 32 +- litellm-rust/crates/cache-gcs/src/token.rs | 2 +- litellm-rust/crates/cache-gcs/tests/cache.rs | 39 +- litellm-rust/crates/cache-memory/src/cache.rs | 48 +- .../crates/cache-memory/tests/cache.rs | 6 +- .../crates/cache-qdrant-semantic/Cargo.toml | 1 + .../crates/cache-qdrant-semantic/src/cache.rs | 14 +- .../cache-qdrant-semantic/src/embedder.rs | 17 +- .../cache-qdrant-semantic/tests/embedder.rs | 21 +- .../cache-qdrant-semantic/tests/qdrant.rs | 26 +- .../crates/cache-redis-semantic/src/cache.rs | 6 +- .../crates/cache-redis-semantic/src/index.rs | 18 +- .../cache-redis-semantic/tests/cache.rs | 30 +- .../cache-redis-semantic/tests/support/mod.rs | 15 +- litellm-rust/crates/cache-redis/Cargo.toml | 2 + litellm-rust/crates/cache-redis/src/cache.rs | 4 +- litellm-rust/crates/cache-redis/src/claim.rs | 14 +- .../crates/cache-redis/src/connection.rs | 74 +-- .../crates/cache-redis/src/counter.rs | 8 +- litellm-rust/crates/cache-redis/src/keys.rs | 4 +- .../crates/cache-redis/src/lifecycle.rs | 9 +- litellm-rust/crates/cache-redis/src/queue.rs | 16 +- litellm-rust/crates/cache-redis/src/script.rs | 2 +- litellm-rust/crates/cache-redis/src/store.rs | 18 +- .../crates/cache-redis/tests/cache.rs | 38 +- .../crates/cache-redis/tests/cluster.rs | 2 +- .../crates/cache-redis/tests/support/mod.rs | 4 +- .../crates/cache-response/src/buffer.rs | 4 +- .../crates/cache-response/src/codec.rs | 28 +- .../crates/cache-response/src/response.rs | 13 +- .../crates/cache-response/tests/codec.rs | 24 +- .../crates/cache-response/tests/response.rs | 12 +- litellm-rust/crates/cache-s3/Cargo.toml | 1 + litellm-rust/crates/cache-s3/src/auth.rs | 2 +- litellm-rust/crates/cache-s3/src/cache.rs | 6 +- litellm-rust/crates/cache-s3/src/transport.rs | 4 +- litellm-rust/crates/cache-s3/tests/cache.rs | 24 +- .../crates/cache-valkey-semantic/Cargo.toml | 2 + .../crates/cache-valkey-semantic/src/index.rs | 39 +- .../cache-valkey-semantic/src/search.rs | 28 +- .../cache-valkey-semantic/tests/cache.rs | 43 +- .../tests/support/mod.rs | 15 +- litellm-rust/crates/cache/Cargo.toml | 1 + litellm-rust/crates/cache/src/capabilities.rs | 4 +- litellm-rust/crates/cache/src/codec.rs | 4 +- litellm-rust/crates/cache/src/dual.rs | 14 +- litellm-rust/crates/cache/src/error.rs | 62 ++- litellm-rust/crates/cache/src/lib.rs | 2 +- litellm-rust/crates/cache/tests/caching.rs | 10 +- litellm-rust/crates/cache/tests/codec.rs | 10 +- litellm-rust/crates/cache/tests/dual.rs | 35 +- litellm-rust/crates/config/Cargo.toml | 1 + litellm-rust/crates/config/src/error.rs | 11 +- litellm-rust/crates/config/src/includes.rs | 35 +- litellm-rust/crates/config/tests/config.rs | 10 +- litellm-rust/crates/core-utils/Cargo.toml | 1 + .../crates/core-utils/src/call_arguments.rs | 19 +- .../src/exception_mapping_utils/mod.rs | 16 +- litellm-rust/crates/core-utils/src/params.rs | 2 + .../crates/core-utils/src/url_utils.rs | 2 +- litellm-rust/crates/cost/Cargo.toml | 3 + litellm-rust/crates/cost/src/error.rs | 27 ++ litellm-rust/crates/cost/src/lib.rs | 88 ++-- litellm-rust/crates/cost/tests/calculation.rs | 20 +- litellm-rust/crates/error/src/lib.rs | 36 +- litellm-rust/crates/framer/src/error.rs | 8 +- litellm-rust/crates/framer/src/framer.rs | 13 - litellm-rust/crates/gateway-auth/Cargo.toml | 1 + .../crates/gateway-auth/tests/auth.rs | 6 +- litellm-rust/crates/host-python/Cargo.toml | 1 + .../crates/host-python/src/binding.rs | 2 +- litellm-rust/crates/host-python/src/driver.rs | 2 +- litellm-rust/crates/host-python/src/error.rs | 8 +- .../crates/host-python/src/marshal.rs | 3 +- .../crates/host-python/tests/support/mod.rs | 2 + litellm-rust/crates/http/Cargo.toml | 1 + litellm-rust/crates/http/src/error.rs | 454 ++++++++++++++---- litellm-rust/crates/http/src/media.rs | 43 +- litellm-rust/crates/http/src/request.rs | 38 +- litellm-rust/crates/http/src/tls.rs | 10 +- litellm-rust/crates/http/src/websocket.rs | 4 +- .../crates/inference-chat/src/handler.rs | 15 +- .../inference-messages/src/common_utils.rs | 16 +- .../crates/inference-messages/src/handler.rs | 8 +- litellm-rust/crates/inference-ocr/Cargo.toml | 1 + .../crates/inference-ocr/src/route.rs | 5 +- .../inference-ocr/tests/ocr/azure_ai.rs | 2 +- .../crates/inference-ocr/tests/ocr/main.rs | 10 +- .../inference-ocr/tests/ocr/vertex_ai.rs | 10 +- .../crates/inference-responses/src/handler.rs | 10 +- .../inference-transcription/src/handler.rs | 10 +- litellm-rust/crates/inference/src/outbound.rs | 8 +- .../crates/inference/tests/caching.rs | 4 +- litellm-rust/crates/llms/Cargo.toml | 1 + .../llms/src/azure_ai/ocr/common_utils.rs | 4 +- .../document_intelligence/transformation.rs | 7 +- .../llms/src/azure_ai/ocr/transformation.rs | 2 +- .../crates/llms/src/base_llm/ocr/error.rs | 4 +- .../messages/transformation.rs | 3 +- .../llms/src/cohere/ocr/transformation.rs | 9 +- .../llms/src/vertex_ai/ocr/common_utils.rs | 4 +- litellm-rust/crates/model-catalog/Cargo.toml | 1 + .../crates/model-catalog/src/catalog.rs | 6 +- .../crates/model-catalog/src/error.rs | 8 +- .../crates/model-catalog/src/validation.rs | 114 +++-- .../tests/registry_validation.rs | 4 +- litellm-rust/crates/python-bridge/Cargo.toml | 1 + .../crates/python-bridge/src/cache/mod.rs | 49 +- .../python-bridge/src/cache/python/host.rs | 6 +- .../python-bridge/src/cache/python/service.rs | 19 +- .../python-bridge/src/cache/selection.rs | 2 +- .../src/routes/clickhouse_spend.rs | 58 ++- .../python-bridge/src/routes/token_counter.rs | 26 +- .../crates/python-bridge/src/secrets/error.rs | 8 +- .../crates/python-bridge/src/tokenizer.rs | 12 +- .../crates/python-compat/src/error.rs | 6 +- .../crates/python-compat/src/pickle.rs | 9 +- litellm-rust/crates/secrets-aws/src/auth.rs | 4 +- litellm-rust/crates/secrets-aws/src/error.rs | 4 +- .../secrets-aws/src/secret_manager/read.rs | 17 +- .../secrets-aws/tests/secret_manager/reads.rs | 6 +- litellm-rust/crates/secrets-azure/Cargo.toml | 1 + .../crates/secrets-azure/src/error.rs | 20 +- .../crates/secrets-azure/src/key_vault.rs | 15 +- .../crates/secrets-azure/tests/key_vault.rs | 18 +- .../crates/secrets-cyberark/Cargo.toml | 1 + .../crates/secrets-cyberark/src/error.rs | 18 +- .../secrets-cyberark/src/secret_manager.rs | 2 +- .../src/secret_manager/client.rs | 17 +- .../src/secret_manager/read.rs | 8 +- .../src/secret_manager/write.rs | 4 +- .../tests/secret_manager/configuration.rs | 12 +- litellm-rust/crates/secrets-google/Cargo.toml | 1 + .../crates/secrets-google/src/error.rs | 38 +- .../secrets-google/src/secret_manager.rs | 20 +- .../secrets-google/tests/secret_manager.rs | 6 +- .../crates/secrets-hashicorp/Cargo.toml | 1 + .../crates/secrets-hashicorp/src/config.rs | 6 +- .../crates/secrets-hashicorp/src/error.rs | 26 +- .../secrets-hashicorp/src/secret_manager.rs | 24 +- .../src/secret_manager/client.rs | 10 +- .../src/secret_manager/raw.rs | 18 +- .../src/secret_manager/write.rs | 2 +- .../tests/secret_manager/configuration.rs | 35 +- .../tests/secret_manager/reads.rs | 2 +- .../crates/secrets-types/tests/cache.rs | 20 +- litellm-rust/crates/secrets/Cargo.toml | 1 + .../crates/secrets/src/compatibility.rs | 6 +- litellm-rust/crates/secrets/src/error.rs | 14 +- litellm-rust/crates/secrets/src/handler.rs | 10 +- litellm-rust/crates/secrets/src/oidc.rs | 20 +- litellm-rust/crates/secrets/tests/aws.rs | 2 +- litellm-rust/crates/secrets/tests/google.rs | 2 +- .../crates/secrets/tests/resolution.rs | 133 ++--- .../crates/spend-clickhouse/Cargo.toml | 1 + .../crates/spend-clickhouse/src/error.rs | 33 +- .../crates/spend-clickhouse/src/insert.rs | 117 +++-- .../crates/spend-clickhouse/src/lib.rs | 4 +- .../crates/storage-clickhouse/Cargo.toml | 2 + .../crates/storage-clickhouse/src/error.rs | 70 ++- .../crates/storage-clickhouse/src/insert.rs | 13 +- .../crates/storage-clickhouse/src/lib.rs | 8 +- .../crates/storage-clickhouse/src/migrate.rs | 24 +- .../crates/storage-clickhouse/src/read.rs | 26 +- .../storage-clickhouse/tests/migrations.rs | 29 +- .../storage-clickhouse/tests/transport.rs | 105 ++-- litellm-rust/crates/testkit/Cargo.toml | 1 + .../crates/testkit/src/agent/configure.rs | 5 +- litellm-rust/crates/testkit/src/error.rs | 14 +- .../crates/testkit/src/install/fetch.rs | 4 +- .../crates/testkit/src/install/mod.rs | 26 +- litellm-rust/crates/testkit/src/session.rs | 12 +- .../crates/token-counter-fast/Cargo.toml | 1 + .../crates/token-counter-fast/src/error.rs | 19 +- .../crates/token-counter-fast/src/tiktoken.rs | 10 +- .../token-counter-huggingface/src/error.rs | 17 +- .../token-counter-huggingface/src/lib.rs | 4 +- .../crates/token-counter-tiktoken/Cargo.toml | 1 + .../token-counter-tiktoken/src/error.rs | 49 +- .../crates/token-counter-tiktoken/src/lib.rs | 29 +- .../token-counter-tiktoken/src/ranks.rs | 47 +- litellm-rust/crates/token-counter/Cargo.toml | 1 + .../crates/token-counter/src/counter.rs | 10 +- .../crates/token-counter/src/error.rs | 91 ++-- litellm-rust/crates/token-counter/src/fast.rs | 12 - .../crates/token-counter/src/huggingface.rs | 11 - litellm-rust/crates/token-counter/src/lib.rs | 2 +- .../crates/token-counter/src/python_json.rs | 16 +- .../crates/token-counter/src/tiktoken.rs | 18 +- .../crates/token-counter/src/tools.rs | 6 +- .../crates/token-counter/src/types.rs | 4 +- .../token-counter/tests/token_counter.rs | 20 +- 228 files changed, 3069 insertions(+), 1742 deletions(-) delete mode 100644 litellm-rust/crates/auth-types/tests/error.rs create mode 100644 litellm-rust/crates/cost/src/error.rs delete mode 100644 litellm-rust/crates/framer/src/framer.rs diff --git a/litellm-rust/.agents/skills/rust-error/references/messages-and-causes.md b/litellm-rust/.agents/skills/rust-error/references/messages-and-causes.md index 81a237219bf..abf2c0cde4a 100644 --- a/litellm-rust/.agents/skills/rust-error/references/messages-and-causes.md +++ b/litellm-rust/.agents/skills/rust-error/references/messages-and-causes.md @@ -66,7 +66,7 @@ Store the lower-level error itself, never its string, and never drop it with `ma | downcasting | the type is gone | | redaction | `reqwest::Error`'s `Display` includes the request URL, which can carry credentials in the path or query | -If the cause is not `Clone` and the error must be, hold it as `litellm_error::Source`, a cloneable `Arc` that is transparent to `Display`, `source()` and `downcast_ref`, or make the crate's error opaque. Do not stringify or drop a cause to get `Clone` or `PartialEq` +If the cause is not `Clone` and the error must be, hold it as `#[source] litellm_error::Source`, a cloneable `Arc` that is transparent to `Display` and `source()`, or make the crate's error opaque. `litellm_error::find::(&error)` walks the chain and looks through `Source`, so it is how anyone reaches a typed cause. A third-party `Box`, or an `anyhow::Error` a dependency returns, converts with `Source::from`, and a dependency's own boxed alias such as `tokenizers::Error` is held as it is. Never put `#[error(transparent)]` on a `Source`, the erased error would then be no node of the chain and `find` could not reach it. Do not stringify or drop a cause to get `Clone` or `PartialEq` ```rust // loses the type and leaks the URL diff --git a/litellm-rust/.agents/skills/rust-error/references/testing.md b/litellm-rust/.agents/skills/rust-error/references/testing.md index b6b4da72d6c..fdf1699e190 100644 --- a/litellm-rust/.agents/skills/rust-error/references/testing.md +++ b/litellm-rust/.agents/skills/rust-error/references/testing.md @@ -20,7 +20,7 @@ In order of preference 2. A real cause, such as `tungstenite::Error::ConnectionClosed` or a `serde_json::from_str("{")` error, passed through our `From` 3. A public constructor, such as `Error::upstream(500, "boom".into())` -Inside the owning crate a test may match the private representation through the `#[cfg(test)] fn repr(&self) -> &Repr` accessor in `crates/http/src/error.rs`, as the `media.rs` and `tls.rs` tests do. Anywhere else, walk `source()` and `downcast_ref` to the cause the test expects +Inside the owning crate a test may match the private representation through the `#[cfg(test)] fn repr(&self) -> &Repr` accessor in `crates/http/src/error.rs`, as the `media.rs` and `tls.rs` tests do. Anywhere else, reach the cause the test expects with `litellm_error::find::(&error)` ## What to cover @@ -28,7 +28,7 @@ Inside the owning crate a test may match the private representation through the |---|---| | classification | one `#[case]` per input class, asserting the predicate | | redaction | build from a URL holding `api_key=secret`, assert the message lacks `secret` and the path | -| cause kept | walk `source()` and `downcast_ref` to the expected type | +| cause kept | `litellm_error::find::(&error)` returns the expected cause | | rendered once | the cause's text appears exactly once in `litellm_error::Report::new(&error).to_string()` and not in `to_string()` | | boundary mapping | the gateway status and body, or the Python exception type and its `status` attribute | diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 461bb212db7..2ca921389cd 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3375,6 +3375,7 @@ dependencies = [ "bytes", "futures-util", "litellm-auth-types", + "litellm-error", "litellm-http", "moka", "reqwest", @@ -3394,6 +3395,7 @@ dependencies = [ "azure_core", "azure_identity", "litellm-auth-types", + "litellm-error", "moka", "rstest", "serde_json", @@ -3412,6 +3414,7 @@ dependencies = [ "google-cloud-auth", "http 1.4.2", "litellm-auth-types", + "litellm-error", "moka", "rstest", "serde", @@ -3426,6 +3429,7 @@ dependencies = [ name = "litellm-auth-types" version = "0.1.0" dependencies = [ + "litellm-error", "rstest", "serde", "serde_json", @@ -3439,6 +3443,7 @@ dependencies = [ name = "litellm-cache" version = "0.1.0" dependencies = [ + "litellm-error", "rstest", "serde", "serde_json", @@ -3464,6 +3469,7 @@ dependencies = [ "reqwest", "rstest", "serde_json", + "thiserror 2.0.19", "tokio", "url", "wiremock", @@ -3482,6 +3488,7 @@ dependencies = [ "serde-pickle", "serde_json", "tempfile", + "thiserror 2.0.19", "tokio", ] @@ -3494,11 +3501,13 @@ dependencies = [ "litellm-auth-types", "litellm-cache", "litellm-cache-testing", + "litellm-error", "litellm-http", "percent-encoding", "reqwest", "rstest", "serde_json", + "thiserror 2.0.19", "tokio", "wiremock", ] @@ -3521,6 +3530,7 @@ dependencies = [ "futures-util", "litellm-cache", "litellm-cache-testing", + "litellm-error", "litellm-http", "qdrant-client", "reqwest", @@ -3541,11 +3551,13 @@ version = "0.1.0" dependencies = [ "litellm-cache", "litellm-cache-testing", + "litellm-error", "r2d2", "redis", "redis-test", "rstest", "serde_json", + "thiserror 2.0.19", "tokio", ] @@ -3598,6 +3610,7 @@ dependencies = [ "litellm-auth-aws", "litellm-cache", "litellm-cache-testing", + "litellm-error", "litellm-http", "reqwest", "rstest", @@ -3620,11 +3633,13 @@ dependencies = [ "litellm-cache", "litellm-cache-redis", "litellm-cache-testing", + "litellm-error", "redis", "redis-test", "rstest", "serde_json", "sha2 0.10.9", + "thiserror 2.0.19", "tokio", "uuid", ] @@ -3648,6 +3663,7 @@ name = "litellm-config" version = "0.1.0" dependencies = [ "litellm-auth-types", + "litellm-error", "litellm-router-types", "rstest", "serde", @@ -3663,6 +3679,7 @@ name = "litellm-core-utils" version = "0.1.0" dependencies = [ "fancy-regex 0.19.2", + "litellm-error", "litellm-llms-types", "litellm-tracing", "rstest", @@ -3690,6 +3707,7 @@ dependencies = [ "criterion", "proptest", "rstest", + "thiserror 2.0.19", ] [[package]] @@ -3777,6 +3795,7 @@ dependencies = [ "futures-util", "litellm-auth-types", "litellm-config", + "litellm-error", "litellm-secrets", "rstest", "serde", @@ -3934,6 +3953,7 @@ dependencies = [ "bytes", "futures-util", "litellm-coroutine", + "litellm-error", "litellm-host", "parking_lot", "pyo3", @@ -3954,6 +3974,7 @@ dependencies = [ "http 1.4.2", "hyper-util", "litellm-core-utils", + "litellm-error", "rcgen", "reqwest", "rstest", @@ -4068,6 +4089,7 @@ dependencies = [ "litellm-auth-aws", "litellm-auth-gcp", "litellm-core-utils", + "litellm-error", "litellm-host", "litellm-host-native", "litellm-http", @@ -4165,6 +4187,7 @@ dependencies = [ "litellm-auth-azure", "litellm-auth-gcp", "litellm-core-utils", + "litellm-error", "litellm-framer", "litellm-host", "litellm-http", @@ -4205,6 +4228,7 @@ version = "0.1.0" dependencies = [ "indexmap 2.14.0", "jsonschema", + "litellm-error", "litellm-llms-types", "rstest", "schemars 1.2.2", @@ -4231,6 +4255,7 @@ dependencies = [ "litellm-cache-response", "litellm-callbacks-legacy-python", "litellm-core-utils", + "litellm-error", "litellm-host", "litellm-host-python", "litellm-http", @@ -4317,6 +4342,7 @@ dependencies = [ "google-cloud-auth", "google-cloud-kms-v1", "litellm-core-utils", + "litellm-error", "litellm-http", "litellm-python-compat", "litellm-secrets-aws", @@ -4375,6 +4401,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.19", "tokio", + "url", "veil", "wiremock", ] @@ -4398,6 +4425,7 @@ dependencies = [ "tempfile", "thiserror 2.0.19", "tokio", + "url", "veil", "wiremock", ] @@ -4424,6 +4452,7 @@ dependencies = [ "serde_json", "thiserror 2.0.19", "tokio", + "url", "veil", "wiremock", ] @@ -4433,6 +4462,7 @@ name = "litellm-secrets-hashicorp" version = "0.1.0" dependencies = [ "litellm-core-utils", + "litellm-error", "litellm-secrets-types", "rstest", "rustify", @@ -4467,6 +4497,7 @@ name = "litellm-spend-clickhouse" version = "0.1.0" dependencies = [ "flate2", + "litellm-error", "litellm-http", "litellm-storage-clickhouse", "rstest", @@ -4487,6 +4518,7 @@ name = "litellm-storage-clickhouse" version = "0.1.0" dependencies = [ "flate2", + "litellm-error", "litellm-http", "rstest", "serde", @@ -4506,6 +4538,7 @@ version = "0.1.0" dependencies = [ "flate2", "futures-util", + "litellm-http", "reqwest", "rstest", "semver", @@ -4529,6 +4562,7 @@ dependencies = [ "criterion", "indexmap 2.14.0", "itoa", + "litellm-error", "litellm-token-counter-fast", "litellm-token-counter-huggingface", "litellm-token-counter-tiktoken", @@ -4545,6 +4579,7 @@ name = "litellm-token-counter-fast" version = "0.1.0" dependencies = [ "base64 0.22.1", + "litellm-error", "rand 0.8.7", "rstest", "rustc-hash", @@ -4570,6 +4605,7 @@ name = "litellm-token-counter-tiktoken" version = "0.1.0" dependencies = [ "base64 0.22.1", + "litellm-error", "once_cell", "rstest", "rustc-hash", diff --git a/litellm-rust/crates/auth-aws/Cargo.toml b/litellm-rust/crates/auth-aws/Cargo.toml index 1a7e2cd6782..e4578777a08 100644 --- a/litellm-rust/crates/auth-aws/Cargo.toml +++ b/litellm-rust/crates/auth-aws/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] litellm-auth-types.workspace = true +litellm-error.workspace = true litellm-http.workspace = true reqwest.workspace = true diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index d83b01c24ad..1ef736e6b55 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -15,13 +15,13 @@ use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use litellm_auth_types::AwsParams; -use super::Error; use super::constants::{ AWS_ACCESS_KEY_ID, AWS_EXTERNAL_ID, AWS_PROFILE_NAME, AWS_REGION_NAME, AWS_ROLE_ARN, AWS_ROLE_NAME, AWS_SECRET_ACCESS_KEY, AWS_SESSION_NAME, AWS_SESSION_TOKEN, AWS_SIGNED_HEADER_NAMES, AWS_STS_ENDPOINT, AWS_WEB_IDENTITY_TOKEN, AWS_WEB_IDENTITY_TOKEN_FILE, DEFAULT_BEDROCK_REGION, DEFAULT_SESSION_NAME_PREFIX, SIGV4_COMPUTED_HEADER_NAMES, }; +use crate::error::{CredentialFlow, Error, SigningStage}; const STATIC_CREDENTIALS_TTL: Duration = Duration::from_secs(3600 - 60); const AMBIENT_CREDENTIALS_TTL: Duration = Duration::from_secs(600); @@ -271,7 +271,7 @@ impl AwsAuthService { provider .provide_credentials() .await - .map_err(|error| Error::AwsProfile(error.to_string())) + .map_err(Error::credentials(CredentialFlow::Profile)) } AwsAuthFlow::AssumeRole { role, session_name } => { if is_already_running_as_role(&role, &resolved).await? { @@ -287,7 +287,7 @@ impl AwsAuthService { let credentials = provider .provide_credentials() .await - .map_err(|error| Error::AwsDefaultChain(error.to_string()))?; + .map_err(Error::credentials(CredentialFlow::DefaultChain))?; self.set_cached_credentials( key, credentials.clone(), @@ -327,7 +327,7 @@ impl AwsAuthService { provider .provide_credentials() .await - .map_err(|error| Error::AwsAssumeRole(error.to_string())) + .map_err(Error::credentials(CredentialFlow::AssumeRole)) } AwsAuthFlow::WebIdentity { token, @@ -350,12 +350,14 @@ impl AwsAuthService { .web_identity_token(token) .send() .await - .map_err(|error| Error::AwsWebIdentity(error.to_string()))?; + .map_err(Error::credentials(CredentialFlow::WebIdentity))?; let credentials = response .credentials() - .ok_or(Error::AwsMissingWebIdentityCredentials)?; - let expiration = SystemTime::try_from(*credentials.expiration()) - .map_err(|error| Error::AwsWebIdentityExpiration(error.to_string()))?; + .ok_or(Error::MissingWebIdentityCredentials)?; + let expiration = + SystemTime::try_from(*credentials.expiration()).map_err(|source| { + Error::WebIdentityExpiration(litellm_error::Source::new(source)) + })?; Ok(Credentials::new( credentials.access_key_id(), credentials.secret_access_key(), @@ -376,7 +378,7 @@ impl AwsAuthService { let credentials = provider .provide_credentials() .await - .map_err(|error| Error::AwsDefaultChain(error.to_string()))?; + .map_err(Error::credentials(CredentialFlow::DefaultChain))?; self.set_cached_credentials( key, credentials.clone(), @@ -480,14 +482,16 @@ pub fn sign_headers( .settings(SigningSettings::default()) .build() .map(SigningParams::from) - .map_err(|error| Error::AwsSigningParameters(error.to_string()))?; + .map_err(Error::signing(SigningStage::Parameters))?; let header_refs = headers .iter() .map(|(name, value)| { value .to_str() .map(|value| (name.as_str(), value)) - .map_err(|_| Error::AwsSignableRequest(format!("{name} is not visible ASCII"))) + .map_err(|_| Error::NonAsciiHeader { + name: name.to_string(), + }) }) .collect::, _>>()?; let request = SignableRequest::new( @@ -496,23 +500,20 @@ pub fn sign_headers( header_refs.into_iter(), SignableBody::Bytes(body), ) - .map_err(|error| Error::AwsSignableRequest(error.to_string()))?; + .map_err(Error::signing(SigningStage::Request))?; let (instructions, _) = sign(request, ¶ms) - .map_err(|error| Error::AwsSigning(error.to_string()))? + .map_err(Error::signing(SigningStage::Signature))? .into_parts(); instructions .headers() .map(|(name, value)| { - HeaderName::from_bytes(name.as_bytes()) - .map_err(|error| error.to_string()) - .and_then(|name| { - HeaderValue::from_str(value) - .map(|value| (name, value)) - .map_err(|error| error.to_string()) - }) + let name = HeaderName::from_bytes(name.as_bytes()) + .map_err(Error::signing(SigningStage::Output))?; + let value = + HeaderValue::from_str(value).map_err(Error::signing(SigningStage::Output))?; + Ok((name, value)) }) - .collect::>() - .map_err(Error::AwsSigning) + .collect() } /// Model-id and region parsing shared by every Bedrock route. @@ -1081,6 +1082,53 @@ mod tests { assert!(!signed.contains_key("X-Amz-Security-Token")); } + #[test] + fn a_header_value_outside_visible_ascii_is_named_instead_of_signed() { + let (url, body, mut headers) = parity_inputs(); + headers.insert("x-amz-meta-note", HeaderValue::from_str("café").unwrap()); + let credentials = Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"); + + let error = sign_headers( + "POST", + &url, + &body, + &headers, + SigningScope { + region: "us-east-1", + service: BEDROCK_SERVICE, + credentials: &credentials, + time: UNIX_EPOCH, + }, + ) + .unwrap_err(); + + assert_eq!( + error, + Error::NonAsciiHeader { + name: "x-amz-meta-note".into() + } + ); + } + + #[test] + fn the_shared_auth_error_nests_the_aws_failure_and_its_cause() { + let cause = std::io::Error::new(std::io::ErrorKind::NotFound, "profile not found"); + let error = + litellm_auth_types::Error::from(Error::credentials(CredentialFlow::Profile)(cause)); + + assert!(matches!( + litellm_error::find::(&error), + Some(Error::Credentials { + flow: CredentialFlow::Profile, + .. + }) + )); + assert_eq!( + litellm_error::Report::new(&error).to_string(), + "provider authentication failed: could not load AWS profile credentials: profile not found" + ); + } + #[ignore] #[tokio::test] async fn live_bedrock_invoke_model_returns_200() -> Result<(), Box> { diff --git a/litellm-rust/crates/auth-aws/src/error.rs b/litellm-rust/crates/auth-aws/src/error.rs index d4c6ae8cf6c..5f88fc8a171 100644 --- a/litellm-rust/crates/auth-aws/src/error.rs +++ b/litellm-rust/crates/auth-aws/src/error.rs @@ -1,46 +1,73 @@ -use thiserror::Error as ThisError; +use litellm_error::Source; -#[derive(Clone, Debug, ThisError, PartialEq, Eq)] +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] pub enum Error { - #[error("AWS profile credentials failed: {0}")] - AwsProfile(String), - #[error("AWS default credentials failed: {0}")] - AwsDefaultChain(String), - #[error("AWS role credentials failed: {0}")] - AwsAssumeRole(String), - #[error("AWS web identity credentials failed: {0}")] - AwsWebIdentity(String), - #[error("AWS web identity expiration was invalid: {0}")] - AwsWebIdentityExpiration(String), - #[error("AWS signing parameters failed: {0}")] - AwsSigningParameters(String), - #[error("AWS signable request failed: {0}")] - AwsSignableRequest(String), - #[error("AWS request signing failed: {0}")] - AwsSigning(String), + #[error("could not load AWS {flow} credentials")] + Credentials { + flow: CredentialFlow, + #[source] + source: Source, + }, #[error("AWS web identity response had no credentials")] - AwsMissingWebIdentityCredentials, + MissingWebIdentityCredentials, + #[error("AWS web identity response had an invalid expiration")] + WebIdentityExpiration(#[source] Source), + #[error("header {name} has a value that is not visible ASCII")] + NonAsciiHeader { name: String }, + #[error("{stage}")] + Signing { + stage: SigningStage, + #[source] + source: Source, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum CredentialFlow { + #[error("profile")] + Profile, + #[error("default chain")] + DefaultChain, + #[error("assumed role")] + AssumeRole, + #[error("web identity")] + WebIdentity, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum SigningStage { + #[error("SigV4 signing parameters are invalid")] + Parameters, + #[error("SigV4 cannot sign this request")] + Request, + #[error("SigV4 could not compute the signature")] + Signature, + #[error("SigV4 produced an invalid header")] + Output, +} + +impl Error { + pub(crate) fn credentials( + flow: CredentialFlow, + ) -> impl FnOnce(E) -> Self { + move |source| Self::Credentials { + flow, + source: Source::new(source), + } + } + + pub(crate) fn signing( + stage: SigningStage, + ) -> impl FnOnce(E) -> Self { + move |source| Self::Signing { + stage, + source: Source::new(source), + } + } } impl From for litellm_auth_types::Error { fn from(error: Error) -> Self { - Self::ProviderAuthentication(error.to_string()) - } -} - -#[cfg(test)] -mod tests { - use super::Error; - - #[test] - fn converts_to_shared_auth_error_without_losing_context() { - let error = litellm_auth_types::Error::from(Error::AwsProfile("profile not found".into())); - - assert_eq!( - error, - litellm_auth_types::Error::ProviderAuthentication( - "AWS profile credentials failed: profile not found".into() - ) - ); + Self::provider(error) } } diff --git a/litellm-rust/crates/auth-aws/src/lib.rs b/litellm-rust/crates/auth-aws/src/lib.rs index 0fe0b390110..bd0f866371f 100644 --- a/litellm-rust/crates/auth-aws/src/lib.rs +++ b/litellm-rust/crates/auth-aws/src/lib.rs @@ -5,5 +5,5 @@ mod signer; pub use aws::*; pub use aws_credential_types::Credentials; -pub use error::Error; +pub use error::{CredentialFlow, Error, SigningStage}; pub use signer::SigV4Signer; diff --git a/litellm-rust/crates/auth-aws/src/signer.rs b/litellm-rust/crates/auth-aws/src/signer.rs index 13818ecf9b9..dff917d7c13 100644 --- a/litellm-rust/crates/auth-aws/src/signer.rs +++ b/litellm-rust/crates/auth-aws/src/signer.rs @@ -73,7 +73,13 @@ impl RequestSigner for SigV4Signer { time: (self.clock)(), }, ) - .map_err(litellm_http::Error::signature) + .map_err(|error| match error { + Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error), + Error::Credentials { .. } + | Error::MissingWebIdentityCredentials + | Error::WebIdentityExpiration(_) + | Error::Signing { .. } => litellm_http::Error::signature(error), + }) } } @@ -187,6 +193,26 @@ mod tests { assert!(error.to_string().contains("authorization"), "{error}"); } + #[test] + fn a_forwarded_header_that_cannot_be_signed_is_the_callers_fault() { + let error = signed( + json( + URL, + &[("x-amz-meta-note".into(), "café".into())], + &json_value!({}), + None, + ) + .unwrap(), + &signer("textract"), + ) + .unwrap_err(); + + let report = litellm_error::Report::new(&error).to_string(); + assert!(report.contains("x-amz-meta-note"), "{report}"); + assert!(error.is_request()); + assert!(!error.dispatched()); + } + #[test] fn a_streaming_body_cannot_be_signed() { let mut request = Request::new(Method::POST, Url::parse(URL).unwrap()); diff --git a/litellm-rust/crates/auth-azure/Cargo.toml b/litellm-rust/crates/auth-azure/Cargo.toml index 1fd9d39d113..f99972f813b 100644 --- a/litellm-rust/crates/auth-azure/Cargo.toml +++ b/litellm-rust/crates/auth-azure/Cargo.toml @@ -18,5 +18,6 @@ azure_core = "1.0.0" azure_identity = { version = "1.0.0", features = ["tokio"] } [dev-dependencies] +litellm-error.workspace = true rstest.workspace = true tokio.workspace = true diff --git a/litellm-rust/crates/auth-azure/src/native.rs b/litellm-rust/crates/auth-azure/src/native.rs index 64752162384..9fda67746d1 100644 --- a/litellm-rust/crates/auth-azure/src/native.rs +++ b/litellm-rust/crates/auth-azure/src/native.rs @@ -12,7 +12,7 @@ use azure_identity::{ }; use sha2::{Digest, Sha256}; -use litellm_auth_types::Error; +use litellm_auth_types::{Error, ErrorDetail}; use litellm_auth_types::{InputSource, ResolvedCredential, SecretValue, Sourced}; use super::credential_provider_cache::{ @@ -133,7 +133,9 @@ impl NativeAzureTokenAcquirer { let token = credential .get_token(&[scope.as_str()], None) .await - .map_err(|error| Error::CredentialAcquisition(error.to_string().into()))?; + .map_err(|error| { + Error::CredentialAcquisition(ErrorDetail::failed("Azure token request", error)) + })?; let expires_on = u64::try_from(token.expires_on.unix_timestamp()) .ok() .map(|seconds| UNIX_EPOCH + Duration::from_secs(seconds)); @@ -250,11 +252,8 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), Error> { let Some(authority) = authority else { return Ok(()); }; - let url = url::Url::parse(authority.value()).map_err(|_| { - Error::InvalidConfiguration( - "Azure authority must be an HTTPS origin without credentials, query, or fragment" - .into(), - ) + let url = url::Url::parse(authority.value()).map_err(|error| { + Error::InvalidConfiguration(ErrorDetail::failed("Azure authority parsing", error)) })?; if url.scheme() != "https" || url.host_str().is_none() @@ -264,10 +263,10 @@ fn validate_authority(request: &NativeAzureRequest) -> Result<(), Error> { || url.fragment().is_some() || !matches!(url.path(), "" | "/") { - return Err(Error::InvalidConfiguration( - "Azure authority must be an HTTPS origin without credentials, query, or fragment" - .into(), - )); + return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType { + field: "Azure authority".into(), + expected: "an HTTPS origin without credentials, query, or fragment", + })); } Ok(()) } @@ -377,7 +376,7 @@ fn trusted_source(sources: &[InputSource]) -> InputSource { fn mixed_sources() -> Result { Err(Error::InvalidConfiguration( - "request-controlled Azure auth inputs cannot be combined with host credentials".into(), + ErrorDetail::MixedInputSources { provider: "Azure" }, )) } @@ -444,7 +443,7 @@ fn build_credential( .map(|credential| credential as Arc), } .map_err(|error| { - Error::InvalidConfiguration(litellm_auth_types::ErrorDetail::failed( + Error::InvalidConfiguration(ErrorDetail::failed( "Azure credential initialization", error, )) @@ -662,12 +661,11 @@ mod tests { )) .unwrap_err(); - assert_eq!( - error, - litellm_auth_types::Error::InvalidConfiguration( + assert!( + litellm_error::Report::new(&error).to_string().contains( "request-controlled Azure auth inputs cannot be combined with host credentials" - .into() - ) + ), + "{error:?}" ); } @@ -695,12 +693,11 @@ mod tests { authority, )) .unwrap_err(); - assert_eq!( - error, - litellm_auth_types::Error::InvalidConfiguration( + assert!( + litellm_error::Report::new(&error).to_string().contains( "Azure authority must be an HTTPS origin without credentials, query, or fragment" - .into() - ) + ), + "{error:?}" ); } } diff --git a/litellm-rust/crates/auth-azure/src/resolve.rs b/litellm-rust/crates/auth-azure/src/resolve.rs index 2142e22db50..44101998dd5 100644 --- a/litellm-rust/crates/auth-azure/src/resolve.rs +++ b/litellm-rust/crates/auth-azure/src/resolve.rs @@ -1,8 +1,8 @@ -use litellm_auth_types::Error; use litellm_auth_types::{ CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, SecretValue, Sourced, TokenProviderHandle, }; +use litellm_auth_types::{Error, ErrorDetail}; use std::future::Future; use std::pin::Pin; use std::sync::Arc; @@ -91,9 +91,9 @@ impl AzureAuthService { AzureCredentialPlan::Caller(caller) => { let credential = caller.acquire().await?; if credential.secret().expose().is_empty() { - return Err(Error::EmptyCallerCredential( - "Azure AD token provider returned an empty token", - )); + return Err(Error::EmptyCallerCredential { + caller: "Azure AD token provider", + }); } Ok(Some(Sourced::new(credential, InputSource::Deployment))) } @@ -106,11 +106,9 @@ impl AzureAuthService { } => { let assertion = resolve_reference(inputs, env_lookup, reference.value()) .await? - .ok_or_else(|| { - Error::CredentialAcquisition( - "Azure OIDC reference did not resolve to a value".into(), - ) - })?; + .ok_or(Error::CredentialAcquisition(ErrorDetail::Unresolved { + subject: "Azure OIDC reference", + }))?; let request = ValidatedAzureRequest::new(NativeAzureRequest::ClientAssertion { tenant_id, client_id, @@ -173,7 +171,12 @@ pub(crate) fn select_auth_plan( .map(|selector| Sourced::new(selector, value.source())) }) .transpose() - .map_err(|_| Error::InvalidConfiguration("invalid Azure credential selector".into()))?; + .map_err(|error| { + Error::InvalidConfiguration(ErrorDetail::failed( + "Azure credential selector parsing", + error, + )) + })?; let federated_token_file = configured_string( &inputs.federated_token_file, AZURE_FEDERATED_TOKEN_FILE_ENV, @@ -263,9 +266,12 @@ fn select_native_plan( let selection_source = selected.source(); match selected.into_value() { - AzureCredentialType::ClientSecretCredential => Err(Error::InvalidConfiguration( - "ClientSecretCredential requires tenant_id, client_id, and client_secret".into(), - )), + AzureCredentialType::ClientSecretCredential => { + Err(Error::InvalidConfiguration(ErrorDetail::MissingSettings { + credential: "ClientSecretCredential", + settings: "tenant_id, client_id, and client_secret", + })) + } AzureCredentialType::WorkloadIdentityCredential => { Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new( workload_request(tenant_id, client_id, federated_token_file, scope, authority)?, @@ -349,22 +355,21 @@ fn workload_request( authority: Option>, ) -> Result { Ok(NativeAzureRequest::WorkloadIdentity { - tenant_id: tenant_id.ok_or_else(|| { - Error::InvalidConfiguration("WorkloadIdentityCredential requires tenant_id".into()) - })?, - client_id: client_id.ok_or_else(|| { - Error::InvalidConfiguration("WorkloadIdentityCredential requires client_id".into()) - })?, - token_file_path: token_file_path.ok_or_else(|| { - Error::InvalidConfiguration( - "WorkloadIdentityCredential requires azure_federated_token_file".into(), - ) - })?, + tenant_id: tenant_id.ok_or(workload_requires("tenant_id"))?, + client_id: client_id.ok_or(workload_requires("client_id"))?, + token_file_path: token_file_path.ok_or(workload_requires("azure_federated_token_file"))?, scope, authority, }) } +fn workload_requires(settings: &'static str) -> Error { + Error::InvalidConfiguration(ErrorDetail::MissingSettings { + credential: "WorkloadIdentityCredential", + settings, + }) +} + fn configured_string( configured: &ConfigValue, environment_name: &str, @@ -410,11 +415,10 @@ async fn resolve_reference( .map_or(CredentialLookup::Missing, CredentialLookup::Found), CredentialRef::None => return Ok(None), CredentialRef::File(_) | CredentialRef::Request(_) | CredentialRef::Host(_) => { - let resolver = inputs.credential_resolver.as_ref().ok_or_else(|| { - Error::InvalidConfiguration( - "credential reference requires a host credential resolver".into(), - ) - })?; + let resolver = inputs + .credential_resolver + .as_ref() + .ok_or(Error::InvalidConfiguration(ErrorDetail::MissingResolver))?; resolver.resolve(reference).await? } }; @@ -433,7 +437,7 @@ fn oidc_reference( let value = token.value().expose(); if token.source() == InputSource::Request && value.starts_with("oidc/") { return Err(Error::InvalidConfiguration( - "request-controlled Azure credential references are not allowed".into(), + ErrorDetail::RequestControlledReference { provider: "Azure" }, )); } if let Some(name) = value.strip_prefix("oidc/env/") { @@ -457,19 +461,17 @@ fn oidc_reference( } if value.starts_with("oidc/") { return Err(Error::InvalidConfiguration( - "unsupported OIDC reference".into(), + ErrorDetail::UnsupportedReference, )); } Ok(None) } -fn non_empty_reference(value: &str, kind: &str) -> Result { +fn non_empty_reference(value: &str, kind: &'static str) -> Result { if value.is_empty() { - return Err(Error::InvalidConfiguration( - litellm_auth_types::ErrorDetail::Empty { - subject: kind.into(), - }, - )); + return Err(Error::InvalidConfiguration(ErrorDetail::Empty { + subject: kind, + })); } Ok(value.to_string()) } @@ -518,7 +520,9 @@ mod tests { expires_on: None, }) } else { - Err(Error::CredentialAcquisition(kind.into())) + Err(Error::CredentialAcquisition( + litellm_auth_types::ErrorDetail::Unresolved { subject: kind }, + )) } }) } @@ -624,7 +628,11 @@ mod tests { ))) .expect_err("unsupported backend must fail validation"); - assert!(error.to_string().contains("unsupported OIDC reference")); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("unsupported OIDC reference") + ); } #[rstest::rstest] @@ -649,11 +657,11 @@ mod tests { }) .unwrap_err(); - assert_eq!( - error, - Error::InvalidConfiguration( - "request-controlled Azure credential references are not allowed".into() - ) + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("request-controlled Azure credential references are not allowed"), + "{error:?}" ); } @@ -715,7 +723,16 @@ mod tests { .await .unwrap_err(); - assert!(matches!(error, Error::CredentialChain(errors) if errors.len() == 2)); + assert!(matches!(&error, Error::CredentialChain(errors) if errors.len() == 2)); + let report = litellm_error::Report::new(&error).to_string(); + assert!( + report.contains("managed-identity did not resolve"), + "{report}" + ); + assert!( + report.contains("developer-tools did not resolve"), + "{report}" + ); } #[derive(Debug)] @@ -763,7 +780,9 @@ mod tests { assert_eq!( error, - Error::EmptyCallerCredential("Azure AD token provider returned an empty token") + Error::EmptyCallerCredential { + caller: "Azure AD token provider" + } ); } } diff --git a/litellm-rust/crates/auth-gcp/Cargo.toml b/litellm-rust/crates/auth-gcp/Cargo.toml index bb4c022580a..7844b35ecc2 100644 --- a/litellm-rust/crates/auth-gcp/Cargo.toml +++ b/litellm-rust/crates/auth-gcp/Cargo.toml @@ -24,5 +24,6 @@ google-cloud-auth = { workspace = true, optional = true } http = { workspace = true, optional = true } [dev-dependencies] +litellm-error.workspace = true rstest.workspace = true tempfile.workspace = true diff --git a/litellm-rust/crates/auth-gcp/src/auth.rs b/litellm-rust/crates/auth-gcp/src/auth.rs index e09a350c012..d3a9e7f3d2c 100644 --- a/litellm-rust/crates/auth-gcp/src/auth.rs +++ b/litellm-rust/crates/auth-gcp/src/auth.rs @@ -234,14 +234,13 @@ enum CredentialCacheKey { Adc, } -const BASE64_CREDENTIALS: &str = "The `vertex_credentials` value is base64-encoded JSON, not the JSON itself. \ - Set it to a readable file path, or to the credentials JSON itself."; - fn load_trusted_service_account(configured: &str) -> Result { if is_base64_encoded_json(configured) { - return Err(Error::InvalidConfiguration(ErrorDetail::Message( - BASE64_CREDENTIALS.into(), - ))); + return Err(Error::InvalidConfiguration( + ErrorDetail::Base64EncodedJson { + field: "vertex_credentials", + }, + )); } if names_a_credentials_file(configured) { CustomServiceAccount::from_file(configured) @@ -299,7 +298,8 @@ mod tests { else { panic!("{configured:?} should not load"); }; - assert!(error.to_string().contains(expected), "{error}"); + let report = litellm_error::Report::new(&error).to_string(); + assert!(report.contains(expected), "{report}"); } #[rstest::rstest] @@ -321,7 +321,7 @@ mod tests { .await { Ok(_) => panic!("{configured:?} should not load"), - Err(error) => error.to_string(), + Err(error) => litellm_error::Report::new(&error).to_string(), } } diff --git a/litellm-rust/crates/auth-gcp/src/sdk.rs b/litellm-rust/crates/auth-gcp/src/sdk.rs index 566df291b7d..6cfeee64d20 100644 --- a/litellm-rust/crates/auth-gcp/src/sdk.rs +++ b/litellm-rust/crates/auth-gcp/src/sdk.rs @@ -55,7 +55,7 @@ impl CredentialsProvider for GoogleCredentials { entity_tag: EntityTag::new(), data, }) - .map_err(|_| CredentialsError::from_msg(false, "Google authentication failed")) + .map_err(|error| CredentialsError::from_source(false, error)) } async fn universe_domain(&self) -> Option { @@ -102,5 +102,9 @@ mod tests { ); let error = credentials.headers(Extensions::new()).await.unwrap_err(); assert!(!format!("{error:?}").contains("private")); + assert_eq!( + std::error::Error::source(&error).and_then(|cause| cause.downcast_ref::()), + Some(&Error::InvalidHeader) + ); } } diff --git a/litellm-rust/crates/auth-types/Cargo.toml b/litellm-rust/crates/auth-types/Cargo.toml index b54c04f0229..400f2640221 100644 --- a/litellm-rust/crates/auth-types/Cargo.toml +++ b/litellm-rust/crates/auth-types/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-error.workspace = true serde.workspace = true serde_json.workspace = true subtle.workspace = true diff --git a/litellm-rust/crates/auth-types/src/credential.rs b/litellm-rust/crates/auth-types/src/credential.rs index 94fb8a58467..4f69c6fffc8 100644 --- a/litellm-rust/crates/auth-types/src/credential.rs +++ b/litellm-rust/crates/auth-types/src/credential.rs @@ -86,9 +86,9 @@ impl CredentialPlan { Self::Caller(caller) => { let credential = caller.acquire().await?; if credential.secret().expose().is_empty() { - return Err(Error::EmptyCallerCredential( - "credential caller returned an empty credential", - )); + return Err(Error::EmptyCallerCredential { + caller: "credential caller", + }); } Ok(CredentialPlanResolution::Resolved(credential)) } @@ -104,8 +104,8 @@ mod tests { CredentialLookup, CredentialLookupFuture, CredentialPlan, CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle, }; - use crate::Error; use crate::SecretValue; + use crate::{Error, ErrorDetail}; #[derive(Debug)] struct HostResolver; @@ -150,9 +150,10 @@ mod tests { impl CredentialResolver for FailingResolver { fn resolve<'a>(&'a self, _reference: &'a CredentialRef) -> CredentialLookupFuture<'a> { Box::pin(async { - Err(Error::CredentialAcquisition( - "host credential lookup failed".into(), - )) + Err(Error::CredentialAcquisition(ErrorDetail::failed( + "host credential lookup", + std::io::Error::from(std::io::ErrorKind::PermissionDenied), + ))) }) } } @@ -168,9 +169,14 @@ mod tests { .await .expect_err("acquisition errors cannot become fallback"); - assert_eq!( - error, - Error::CredentialAcquisition("host credential lookup failed".into()) + assert!( + matches!(error, Error::CredentialAcquisition(_)), + "{error:?}" + ); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("host credential lookup failed") ); } } diff --git a/litellm-rust/crates/auth-types/src/error.rs b/litellm-rust/crates/auth-types/src/error.rs index 4986ce3e787..3e93bf9e737 100644 --- a/litellm-rust/crates/auth-types/src/error.rs +++ b/litellm-rust/crates/auth-types/src/error.rs @@ -1,16 +1,18 @@ -use thiserror::Error as ThisError; +use litellm_error::{Report, Source}; -#[derive(Clone, Debug, ThisError, PartialEq, Eq)] +use crate::{ParamKind, ParamSpec}; + +#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)] pub enum Error { - #[error("invalid authentication configuration: {0}")] - InvalidConfiguration(ErrorDetail), - #[error("credential acquisition failed: {0}")] - CredentialAcquisition(ErrorDetail), - #[error("credential caller failed: {0}")] - EmptyCallerCredential(&'static str), - #[error("{0}")] - ProviderAuthentication(String), - #[error("credential acquisition failed: {}", .0.iter().map(ToString::to_string).collect::>().join("; "))] + #[error("invalid authentication configuration")] + InvalidConfiguration(#[source] ErrorDetail), + #[error("credential acquisition failed")] + CredentialAcquisition(#[source] ErrorDetail), + #[error("{caller} returned an empty credential")] + EmptyCallerCredential { caller: &'static str }, + #[error("provider authentication failed")] + Provider(#[source] Source), + #[error("no credential in the chain succeeded: {}", reports(.0))] CredentialChain(Vec), #[error( "Missing {provider} API Key - Set `api_key` or the {environment_variable} environment variable" @@ -22,7 +24,7 @@ pub enum Error { #[error("Missing {provider} {} - {}", .spec.setting, .spec.guidance())] MissingParam { provider: &'static str, - spec: &'static crate::ParamSpec, + spec: &'static ParamSpec, }, #[error("Missing {provider} API Base - {guidance}")] MissingApiBase { @@ -33,31 +35,97 @@ pub enum Error { InvalidHeader, } +fn reports(failures: &[Error]) -> String { + failures + .iter() + .map(|failure| Report::new(failure).to_string()) + .collect::>() + .join("; ") +} + impl Error { + pub fn provider(cause: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Provider(Source::new(cause)) + } + /// The environment lacks the credential the provider needs, as opposed to the caller /// having sent a bad one. pub fn is_missing_credential(&self) -> bool { - matches!(self, Self::MissingApiKey { .. }) + match self { + Self::MissingApiKey { .. } => true, + Self::MissingParam { spec, .. } => match spec.kind { + ParamKind::Credential => true, + ParamKind::Setting => false, + }, + Self::InvalidConfiguration(_) + | Self::CredentialAcquisition(_) + | Self::EmptyCallerCredential { .. } + | Self::Provider(_) + | Self::CredentialChain(_) + | Self::MissingApiBase { .. } + | Self::InvalidHeader => false, + } + } + + /// A connection setting the caller can pass on the call, such as a region, a project or an + /// API base, is unset everywhere it is read from. + pub fn is_missing_setting(&self) -> bool { + match self { + Self::MissingApiBase { .. } => true, + Self::MissingParam { spec, .. } => match spec.kind { + ParamKind::Setting => true, + ParamKind::Credential => false, + }, + Self::InvalidConfiguration(_) + | Self::CredentialAcquisition(_) + | Self::EmptyCallerCredential { .. } + | Self::Provider(_) + | Self::CredentialChain(_) + | Self::MissingApiKey { .. } + | Self::InvalidHeader => false, + } } } #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum ErrorDetail { - #[error("{0}")] - Message(String), #[error("{field} must be {expected}")] InvalidType { field: String, expected: &'static str, }, #[error("{subject} cannot be empty")] - Empty { subject: String }, + Empty { subject: &'static str }, #[error("credential header {0} already exists")] DuplicateHeader(&'static str), - #[error("{operation} failed: {cause}")] + #[error("credential plan is not allowed by the provider auth policy")] + PlanNotAllowed, + #[error("{credential} requires {settings}")] + MissingSettings { + credential: &'static str, + settings: &'static str, + }, + #[error("credentials cannot be sent to a request-controlled {provider} endpoint")] + RequestControlledEndpoint { provider: &'static str }, + #[error("request-controlled {provider} credential references are not allowed")] + RequestControlledReference { provider: &'static str }, + #[error("request-controlled {provider} auth inputs cannot be combined with host credentials")] + MixedInputSources { provider: &'static str }, + #[error("unsupported OIDC reference")] + UnsupportedReference, + #[error("credential reference requires a host credential resolver")] + MissingResolver, + #[error("{subject} did not resolve to a value")] + Unresolved { subject: &'static str }, + #[error( + "The `{field}` value is base64-encoded JSON, not the JSON itself. Set it to a readable file path, or to the credentials JSON itself." + )] + Base64EncodedJson { field: &'static str }, + #[error("{operation} failed")] Failed { operation: &'static str, - cause: ErrorSource, + #[source] + cause: Source, }, } @@ -68,50 +136,151 @@ impl ErrorDetail { ) -> Self { Self::Failed { operation, - cause: ErrorSource::new(cause), + cause: Source::new(cause), } } } -impl From for ErrorDetail { - fn from(message: String) -> Self { - Self::Message(message) +#[cfg(test)] +mod tests { + use std::io; + + use litellm_error::{Report, Source}; + use rstest::rstest; + + use super::{Error, ErrorDetail}; + use crate::{AwsParams, VertexParams}; + + #[rstest] + #[case::api_key( + Error::MissingApiKey { provider: "Example", environment_variable: "EXAMPLE_API_KEY" }, + "Missing Example API Key - Set `api_key` or the EXAMPLE_API_KEY environment variable" + )] + #[case::another_api_key( + Error::MissingApiKey { provider: "Custom", environment_variable: "CUSTOM_KEY" }, + "Missing Custom API Key - Set `api_key` or the CUSTOM_KEY environment variable" + )] + #[case::api_base( + Error::MissingApiBase { provider: "Example", guidance: "Pass api_base" }, + "Missing Example API Base - Pass api_base" + )] + #[case::another_api_base( + Error::MissingApiBase { provider: "Custom", guidance: "Set CUSTOM_ENDPOINT" }, + "Missing Custom API Base - Set CUSTOM_ENDPOINT" + )] + fn missing_inputs_keep_the_python_text(#[case] error: Error, #[case] expected: &str) { + assert_eq!(error.to_string(), expected); + } + + #[rstest] + #[case::configuration( + Error::InvalidConfiguration(ErrorDetail::Empty { subject: "credential selector" }), + "invalid authentication configuration: credential selector cannot be empty" + )] + #[case::acquisition( + Error::CredentialAcquisition(ErrorDetail::Unresolved { subject: "OIDC reference" }), + "credential acquisition failed: OIDC reference did not resolve to a value" + )] + #[case::empty_caller( + Error::EmptyCallerCredential { caller: "token provider" }, + "token provider returned an empty credential" + )] + #[case::chain( + Error::CredentialChain(vec![ + Error::CredentialAcquisition(ErrorDetail::Unresolved { subject: "OIDC reference" }), + Error::EmptyCallerCredential { caller: "token provider" }, + ]), + "no credential in the chain succeeded: credential acquisition failed: OIDC reference did not resolve \ + to a value; token provider returned an empty credential" + )] + fn the_report_names_every_layer_in_order(#[case] error: Error, #[case] expected: &str) { + assert!( + Report::new(&error).to_string().contains(expected), + "{}", + Report::new(&error) + ); + } + + #[rstest] + #[case::configuration(Error::InvalidConfiguration(ErrorDetail::failed( + "test credential", + denied() + )))] + #[case::acquisition(Error::CredentialAcquisition(ErrorDetail::failed( + "test credential", + denied() + )))] + fn a_failure_keeps_its_cause_as_the_source_and_renders_it_once(#[case] error: Error) { + let cause = + std::iter::successors(std::error::Error::source(&error), |cause| cause.source()) + .find_map(|cause| cause.downcast_ref::()?.downcast_ref::()) + .expect("the io cause is in the source chain"); + assert_eq!(cause.kind(), io::ErrorKind::PermissionDenied); + let report = Report::new(&error).to_string(); + assert_eq!(report.matches("access denied").count(), 1, "{report}"); + } + + #[rstest] + #[case::configuration(Error::InvalidConfiguration(ErrorDetail::failed( + "test credential", + denied() + )))] + #[case::acquisition(Error::CredentialAcquisition(ErrorDetail::failed( + "test credential", + denied() + )))] + fn a_contextual_failure_leaves_its_cause_out_of_its_own_message(#[case] error: Error) { + assert!(!error.to_string().contains("access denied"), "{error}"); + assert!( + Report::new(&error) + .to_string() + .contains("test credential failed: access denied") + ); + } + + #[test] + fn a_provider_failure_keeps_the_provider_error_as_its_source() { + let error = Error::provider(denied()); + assert_eq!( + litellm_error::find::(&error).map(io::Error::kind), + Some(io::ErrorKind::PermissionDenied) + ); + assert_eq!( + Report::new(&error).to_string(), + "provider authentication failed: access denied" + ); + } + + fn denied() -> io::Error { + io::Error::new(io::ErrorKind::PermissionDenied, "access denied") + } + + #[rstest] + #[case::api_key(Error::MissingApiKey { provider: "Example", environment_variable: "EXAMPLE_API_KEY" }, true)] + #[case::vertex_credentials(VertexParams::CREDENTIALS.missing("Vertex AI"), true)] + #[case::aws_secret_access_key(AwsParams::SECRET_ACCESS_KEY.missing("AWS"), true)] + #[case::vertex_project(VertexParams::PROJECT.missing("Vertex AI"), false)] + #[case::aws_region(AwsParams::REGION.missing("AWS"), false)] + #[case::api_base(Error::MissingApiBase { provider: "Example", guidance: "Pass api_base" }, false)] + #[case::invalid_configuration(Error::InvalidConfiguration(ErrorDetail::PlanNotAllowed), false)] + fn only_an_absent_credential_is_a_missing_credential( + #[case] error: Error, + #[case] expected: bool, + ) { + assert_eq!(error.is_missing_credential(), expected); + } + + #[rstest] + #[case::vertex_project(VertexParams::PROJECT.missing("Vertex AI"), true)] + #[case::aws_region(AwsParams::REGION.missing("AWS"), true)] + #[case::api_base(Error::MissingApiBase { provider: "Example", guidance: "Pass api_base" }, true)] + #[case::vertex_credentials(VertexParams::CREDENTIALS.missing("Vertex AI"), false)] + #[case::api_key(Error::MissingApiKey { provider: "Example", environment_variable: "EXAMPLE_API_KEY" }, false)] + #[case::invalid_configuration(Error::InvalidConfiguration(ErrorDetail::PlanNotAllowed), false)] + fn an_absent_connection_setting_is_a_missing_setting( + #[case] error: Error, + #[case] expected: bool, + ) { + assert_eq!(error.is_missing_setting(), expected); } } - -impl From<&str> for ErrorDetail { - fn from(message: &str) -> Self { - Self::Message(message.into()) - } -} - -#[derive(Clone, Debug)] -pub struct ErrorSource(std::sync::Arc); - -impl std::ops::Deref for ErrorSource { - type Target = dyn std::error::Error + Send + Sync; - - fn deref(&self) -> &Self::Target { - self.0.as_ref() - } -} - -impl std::fmt::Display for ErrorSource { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(&self.0, formatter) - } -} - -impl ErrorSource { - pub fn new(error: impl std::error::Error + Send + Sync + 'static) -> Self { - Self(std::sync::Arc::new(error)) - } -} - -impl PartialEq for ErrorSource { - fn eq(&self, other: &Self) -> bool { - std::sync::Arc::ptr_eq(&self.0, &other.0) - } -} - -impl Eq for ErrorSource {} diff --git a/litellm-rust/crates/auth-types/src/http.rs b/litellm-rust/crates/auth-types/src/http.rs index 2de3bad9ecf..2ad07f7d541 100644 --- a/litellm-rust/crates/auth-types/src/http.rs +++ b/litellm-rust/crates/auth-types/src/http.rs @@ -21,9 +21,9 @@ pub fn apply_credential( placement: CredentialPlacement, ) -> Result, Error> { if credential.trim().is_empty() { - return Err(Error::InvalidConfiguration( - "credential cannot be empty".into(), - )); + return Err(Error::InvalidConfiguration(crate::ErrorDetail::Empty { + subject: "credential", + })); } if headers .iter() @@ -71,6 +71,10 @@ mod tests { ) .expect_err("provider policy must handle existing credentials"); - assert!(error.to_string().contains("already exists")); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("already exists") + ); } } diff --git a/litellm-rust/crates/auth-types/src/lib.rs b/litellm-rust/crates/auth-types/src/lib.rs index c875bda7979..99651977867 100644 --- a/litellm-rust/crates/auth-types/src/lib.rs +++ b/litellm-rust/crates/auth-types/src/lib.rs @@ -51,9 +51,9 @@ pub use credential::{ CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan, CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle, }; -pub use error::{Error, ErrorDetail, ErrorSource}; +pub use error::{Error, ErrorDetail}; pub use http::CredentialPlacement; -pub use params::{AwsParams, ParamSpec, VertexParams}; +pub use params::{AwsParams, ParamKind, ParamSpec, VertexParams}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use secret::SecretValue; pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; diff --git a/litellm-rust/crates/auth-types/src/params/aws.rs b/litellm-rust/crates/auth-types/src/params/aws.rs index 25fe52b2ce1..8075fc15e9f 100644 --- a/litellm-rust/crates/auth-types/src/params/aws.rs +++ b/litellm-rust/crates/auth-types/src/params/aws.rs @@ -4,15 +4,17 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use veil::Redact; -use super::ParamSpec; +use super::{ParamKind, ParamSpec}; const fn spec( setting: &'static str, + kind: ParamKind, wire: &'static [&'static str], env: &'static [&'static str], ) -> ParamSpec { ParamSpec { setting, + kind, wire, module_global: None, env, @@ -53,40 +55,67 @@ pub struct AwsParams { impl AwsParams { pub const ACCESS_KEY_ID: ParamSpec = spec( "access key id", + ParamKind::Credential, &["aws_access_key_id"], &["AWS_ACCESS_KEY_ID"], ); pub const SECRET_ACCESS_KEY: ParamSpec = spec( "secret access key", + ParamKind::Credential, &["aws_secret_access_key"], &["AWS_SECRET_ACCESS_KEY"], ); pub const SESSION_TOKEN: ParamSpec = spec( "session token", + ParamKind::Credential, &["aws_session_token"], &["AWS_SESSION_TOKEN"], ); pub const REGION: ParamSpec = spec( "region", + ParamKind::Setting, &["aws_region_name"], &["AWS_REGION_NAME", "AWS_REGION"], ); - pub const SESSION_NAME: ParamSpec = - spec("session name", &["aws_session_name"], &["AWS_SESSION_NAME"]); - pub const PROFILE_NAME: ParamSpec = - spec("profile name", &["aws_profile_name"], &["AWS_PROFILE_NAME"]); - pub const ROLE_NAME: ParamSpec = spec("role name", &["aws_role_name"], &["AWS_ROLE_NAME"]); + pub const SESSION_NAME: ParamSpec = spec( + "session name", + ParamKind::Setting, + &["aws_session_name"], + &["AWS_SESSION_NAME"], + ); + pub const PROFILE_NAME: ParamSpec = spec( + "profile name", + ParamKind::Setting, + &["aws_profile_name"], + &["AWS_PROFILE_NAME"], + ); + pub const ROLE_NAME: ParamSpec = spec( + "role name", + ParamKind::Setting, + &["aws_role_name"], + &["AWS_ROLE_NAME"], + ); pub const WEB_IDENTITY_TOKEN: ParamSpec = spec( "web identity token", + ParamKind::Credential, &["aws_web_identity_token"], &["AWS_WEB_IDENTITY_TOKEN"], ); - pub const STS_ENDPOINT: ParamSpec = - spec("STS endpoint", &["aws_sts_endpoint"], &["AWS_STS_ENDPOINT"]); - pub const EXTERNAL_ID: ParamSpec = - spec("external id", &["aws_external_id"], &["AWS_EXTERNAL_ID"]); + pub const STS_ENDPOINT: ParamSpec = spec( + "STS endpoint", + ParamKind::Setting, + &["aws_sts_endpoint"], + &["AWS_STS_ENDPOINT"], + ); + pub const EXTERNAL_ID: ParamSpec = spec( + "external id", + ParamKind::Setting, + &["aws_external_id"], + &["AWS_EXTERNAL_ID"], + ); pub const BEDROCK_RUNTIME_ENDPOINT: ParamSpec = spec( "Bedrock runtime endpoint", + ParamKind::Setting, &["aws_bedrock_runtime_endpoint"], &["AWS_BEDROCK_RUNTIME_ENDPOINT"], ); diff --git a/litellm-rust/crates/auth-types/src/params/mod.rs b/litellm-rust/crates/auth-types/src/params/mod.rs index 88bc64cc2fc..daad1aff8a8 100644 --- a/litellm-rust/crates/auth-types/src/params/mod.rs +++ b/litellm-rust/crates/auth-types/src/params/mod.rs @@ -10,11 +10,18 @@ pub use vertex::VertexParams; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct ParamSpec { pub setting: &'static str, + pub kind: ParamKind, pub wire: &'static [&'static str], pub module_global: Option<&'static str>, pub env: &'static [&'static str], } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParamKind { + Credential, + Setting, +} + impl ParamSpec { /// The first non-blank value: each wire name through `params`, then each environment /// name through `env`. diff --git a/litellm-rust/crates/auth-types/src/params/vertex.rs b/litellm-rust/crates/auth-types/src/params/vertex.rs index 06ec3008dfb..2a6b8bc0ca6 100644 --- a/litellm-rust/crates/auth-types/src/params/vertex.rs +++ b/litellm-rust/crates/auth-types/src/params/vertex.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Deserializer, Serialize, de::Unexpected}; use serde_json::Value; use veil::Redact; -use super::ParamSpec; +use super::{ParamKind, ParamSpec}; /// The Vertex AI fields of Python's `GenericLiteLLMParams`, both the current names and the /// `vertex_ai_*` spellings `VertexBase.safe_get_vertex_ai_*` still read. @@ -27,18 +27,21 @@ pub struct VertexParams { impl VertexParams { pub const CREDENTIALS: ParamSpec = ParamSpec { setting: "credentials", + kind: ParamKind::Credential, wire: &["vertex_credentials", "vertex_ai_credentials"], module_global: None, env: &["VERTEXAI_CREDENTIALS"], }; pub const PROJECT: ParamSpec = ParamSpec { setting: "project", + kind: ParamKind::Setting, wire: &["vertex_project", "vertex_ai_project"], module_global: Some("vertex_project"), env: &["VERTEXAI_PROJECT"], }; pub const LOCATION: ParamSpec = ParamSpec { setting: "location", + kind: ParamKind::Setting, wire: &["vertex_location", "vertex_ai_location"], module_global: Some("vertex_location"), env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"], diff --git a/litellm-rust/crates/auth-types/src/policy.rs b/litellm-rust/crates/auth-types/src/policy.rs index ee2bf08bede..8fff59e284a 100644 --- a/litellm-rust/crates/auth-types/src/policy.rs +++ b/litellm-rust/crates/auth-types/src/policy.rs @@ -1,5 +1,5 @@ use crate::http::apply_credential; -use crate::{CredentialPlacement, Error, ResolvedCredential}; +use crate::{CredentialPlacement, Error, ErrorDetail, ResolvedCredential}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CredentialPlanKind { @@ -31,11 +31,18 @@ pub struct ProviderAuthPolicy { impl ProviderAuthPolicy { pub fn has_existing_credential(&self, headers: &[(String, String)]) -> bool { - headers.iter().any(|(name, _)| { - self.accepted_existing_headers - .iter() - .any(|accepted| name.eq_ignore_ascii_case(accepted)) - }) + self.existing_credential_header(headers).is_some() + } + + fn existing_credential_header(&self, headers: &[(String, String)]) -> Option<&'static str> { + self.accepted_existing_headers + .iter() + .copied() + .find(|accepted| { + headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case(accepted)) + }) } pub fn apply( @@ -44,11 +51,11 @@ impl ProviderAuthPolicy { kind: CredentialPlanKind, credential: &ResolvedCredential, ) -> Result, Error> { - if self.has_existing_credential(&headers) { + if let Some(existing) = self.existing_credential_header(&headers) { return match self.existing_header_behavior { ExistingHeaderBehavior::Preserve => Ok(headers), ExistingHeaderBehavior::Reject => Err(Error::InvalidConfiguration( - "credential header already exists".into(), + ErrorDetail::DuplicateHeader(existing), )), }; } @@ -56,11 +63,7 @@ impl ProviderAuthPolicy { .rules .iter() .find(|rule| rule.kind == kind) - .ok_or_else(|| { - Error::InvalidConfiguration( - "credential plan is not allowed by the provider auth policy".into(), - ) - })?; + .ok_or(Error::InvalidConfiguration(ErrorDetail::PlanNotAllowed))?; apply_credential(headers, credential.secret().expose(), rule.placement) } } @@ -108,6 +111,10 @@ mod tests { ) .unwrap_err(); - assert!(error.to_string().contains("not allowed")); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("not allowed") + ); } } diff --git a/litellm-rust/crates/auth-types/tests/error.rs b/litellm-rust/crates/auth-types/tests/error.rs deleted file mode 100644 index 24865922857..00000000000 --- a/litellm-rust/crates/auth-types/tests/error.rs +++ /dev/null @@ -1,81 +0,0 @@ -use litellm_auth_types::Error; -use rstest::rstest; - -#[rstest] -#[case::api_key( - Error::MissingApiKey { provider: "Example", environment_variable: "EXAMPLE_API_KEY" }, - "Missing Example API Key - Set `api_key` or the EXAMPLE_API_KEY environment variable" -)] -#[case::another_api_key( - Error::MissingApiKey { provider: "Custom", environment_variable: "CUSTOM_KEY" }, - "Missing Custom API Key - Set `api_key` or the CUSTOM_KEY environment variable" -)] -#[case::api_base( - Error::MissingApiBase { provider: "Example", guidance: "Pass api_base" }, - "Missing Example API Base - Pass api_base" -)] -#[case::another_api_base( - Error::MissingApiBase { provider: "Custom", guidance: "Set CUSTOM_ENDPOINT" }, - "Missing Custom API Base - Set CUSTOM_ENDPOINT" -)] -#[case::configuration( - Error::InvalidConfiguration("credential selector is invalid".into()), - "invalid authentication configuration: credential selector is invalid" -)] -#[case::acquisition( - Error::CredentialAcquisition("token expired".into()), - "credential acquisition failed: token expired" -)] -#[case::caller( - Error::EmptyCallerCredential("empty token"), - "credential caller failed: empty token" -)] -#[case::provider( - Error::ProviderAuthentication("provider rejected credentials".into()), - "provider rejected credentials" -)] -#[case::chain( - Error::CredentialChain(vec![ - Error::CredentialAcquisition("token expired".into()), - Error::EmptyCallerCredential("empty token"), - ]), - "credential acquisition failed: credential acquisition failed: token expired; credential caller failed: empty token" -)] -fn display_preserves_failure_phase_and_caller_context( - #[case] error: Error, - #[case] expected: &str, -) { - assert_eq!(error.to_string(), expected); -} - -#[rstest] -#[case::configuration(true)] -#[case::acquisition(false)] -fn contextual_failures_render_their_cause_once(#[case] configuration: bool) { - use litellm_auth_types::ErrorDetail; - - let cause = std::io::Error::from(std::io::ErrorKind::PermissionDenied); - let cause_message = cause.to_string(); - let detail = ErrorDetail::failed("test credential", cause); - let error = if configuration { - Error::InvalidConfiguration(detail) - } else { - Error::CredentialAcquisition(detail) - }; - let (Error::InvalidConfiguration(detail) | Error::CredentialAcquisition(detail)) = &error - else { - unreachable!("built above"); - }; - let ErrorDetail::Failed { cause, .. } = detail else { - panic!("a contextual failure keeps its typed cause: {detail:?}"); - }; - assert_eq!( - cause - .downcast_ref::() - .map(std::io::Error::kind), - Some(std::io::ErrorKind::PermissionDenied) - ); - assert!(error.to_string().contains("test credential failed:")); - assert!(error.to_string().ends_with(&cause_message)); - assert!(std::error::Error::source(&error).is_none()); -} diff --git a/litellm-rust/crates/auth-types/tests/params.rs b/litellm-rust/crates/auth-types/tests/params.rs index 9d488b170e1..38890edb39d 100644 --- a/litellm-rust/crates/auth-types/tests/params.rs +++ b/litellm-rust/crates/auth-types/tests/params.rs @@ -1,8 +1,9 @@ -use litellm_auth_types::{AwsParams, ParamSpec}; +use litellm_auth_types::{AwsParams, ParamKind, ParamSpec}; use rstest::rstest; const LOCATION: ParamSpec = ParamSpec { setting: "location", + kind: ParamKind::Setting, wire: &["vertex_location", "vertex_ai_location"], module_global: Some("vertex_location"), env: &["VERTEXAI_LOCATION", "VERTEX_LOCATION"], diff --git a/litellm-rust/crates/cache-azure-blob/Cargo.toml b/litellm-rust/crates/cache-azure-blob/Cargo.toml index c28cb90d84a..202300c7ab4 100644 --- a/litellm-rust/crates/cache-azure-blob/Cargo.toml +++ b/litellm-rust/crates/cache-azure-blob/Cargo.toml @@ -16,6 +16,7 @@ azure_core = "1.1.0" azure_storage_blob = "1.1.0" futures-util.workspace = true reqwest.workspace = true +thiserror.workspace = true tokio.workspace = true url.workspace = true diff --git a/litellm-rust/crates/cache-azure-blob/src/cache.rs b/litellm-rust/crates/cache-azure-blob/src/cache.rs index c5c1fdd8ab9..e3b013eba4f 100644 --- a/litellm-rust/crates/cache-azure-blob/src/cache.rs +++ b/litellm-rust/crates/cache-azure-blob/src/cache.rs @@ -18,6 +18,10 @@ use url::Url; use crate::{credential::AzureBlobCredential, transport::ReqwestTransport}; +#[derive(Debug, thiserror::Error)] +#[error("account URL cannot hold a container path")] +struct NotABaseUrl; + pub struct AzureBlobCache { container: BlobContainerClient, codec: C, @@ -57,12 +61,12 @@ impl AzureBlobCache { codec: C, runtime: Handle, ) -> Result { - let parsed = Url::parse(account_url).map_err(|_| Error::Unavailable)?; + let parsed = Url::parse(account_url).map_err(Error::unavailable)?; let account_url = parsed.as_str().trim_end_matches('/').to_string(); let container_url = { let mut url = parsed; url.path_segments_mut() - .map_err(|()| Error::Unavailable)? + .map_err(|()| Error::unavailable(NotABaseUrl))? .pop_if_empty() .push(container); url @@ -75,7 +79,7 @@ impl AzureBlobCache { ..BlobContainerClientOptions::default() }), ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let cache = Self { container: client, codec, @@ -101,7 +105,7 @@ impl AzureBlobCache { Err(error) if is_storage_error(&error, StorageErrorCode::ContainerAlreadyExists) => { Ok(()) } - Err(_) => Err(Error::Unavailable), + Err(error) => Err(Error::unavailable(error)), } } @@ -116,7 +120,7 @@ impl AzureBlobCache { { Ok(_) => Ok(()), Err(error) if !overwrite && is_already_present(&error) => Ok(()), - Err(_) => Err(Error::Unavailable), + Err(error) => Err(Error::unavailable(error)), } } @@ -126,13 +130,9 @@ impl AzureBlobCache { Err(error) if is_storage_error(&error, StorageErrorCode::BlobNotFound) => { return Ok(None); } - Err(_) => return Err(Error::Unavailable), + Err(error) => return Err(Error::unavailable(error)), }; - let bytes = response - .body - .collect() - .await - .map_err(|_| Error::Unavailable)?; + let bytes = response.body.collect().await.map_err(Error::unavailable)?; self.codec.decode(&bytes).map(Some) } @@ -140,16 +140,16 @@ impl AzureBlobCache { let mut pages = self .container .list_blobs(None) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .into_pages(); - while let Some(page) = pages.try_next().await.map_err(|_| Error::Unavailable)? { - let page = page.into_model().map_err(|_| Error::Unavailable)?; + while let Some(page) = pages.try_next().await.map_err(Error::unavailable)? { + let page = page.into_model().map_err(Error::unavailable)?; for name in page.blob_items.into_iter().filter_map(|item| item.name) { self.container .blob_client(&name) .delete(None) .await - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } } Ok(()) diff --git a/litellm-rust/crates/cache-azure-blob/src/credential.rs b/litellm-rust/crates/cache-azure-blob/src/credential.rs index d1a3d0e44ec..f58a599f8e3 100644 --- a/litellm-rust/crates/cache-azure-blob/src/credential.rs +++ b/litellm-rust/crates/cache-azure-blob/src/credential.rs @@ -17,6 +17,10 @@ const LLM_TOKEN_ENV: &str = "AZURE_AD_TOKEN"; type EnvLookup = Arc Option + Send + Sync>; +#[derive(Debug, thiserror::Error)] +#[error("no Azure credential is available for blob storage")] +struct NoCredential; + pub struct AzureBlobCredential { service: AzureAuthService, env_lookup: EnvLookup, @@ -62,15 +66,8 @@ impl TokenCredential for AzureBlobCredential { &lookup, ) .await - .map_err(|error| { - azure_core::Error::with_message(ErrorKind::Credential, error.to_string()) - })? - .ok_or_else(|| { - azure_core::Error::with_message( - ErrorKind::Credential, - "no Azure credential is available for blob storage", - ) - })?; + .map_err(|error| azure_core::Error::new(ErrorKind::Credential, error))? + .ok_or_else(|| azure_core::Error::new(ErrorKind::Credential, NoCredential))?; let (token, expires_on) = match credential.into_value() { ResolvedCredential::AccessToken { token, expires_on } => (token, expires_on), ResolvedCredential::Static(token) => (token, None), diff --git a/litellm-rust/crates/cache-azure-blob/src/transport.rs b/litellm-rust/crates/cache-azure-blob/src/transport.rs index 3914b92365c..e3f2cce8700 100644 --- a/litellm-rust/crates/cache-azure-blob/src/transport.rs +++ b/litellm-rust/crates/cache-azure-blob/src/transport.rs @@ -29,7 +29,7 @@ impl HttpClient for ReqwestTransport { } else { ErrorKind::Io }; - azure_core::Error::new(kind, error) + azure_core::Error::new(kind, litellm_http::Error::from(error)) })?; let status = response.status().as_u16().into(); let mut headers = Headers::new(); @@ -41,9 +41,9 @@ impl HttpClient for ReqwestTransport { ); } } - let body = response - .bytes_stream() - .map_err(|error| azure_core::Error::new(ErrorKind::Io, error)); + let body = response.bytes_stream().map_err(|error| { + azure_core::Error::new(ErrorKind::Io, litellm_http::Error::from(error)) + }); Ok(AsyncRawResponse::new(status, headers, Box::pin(body))) } } diff --git a/litellm-rust/crates/cache-azure-blob/tests/cache.rs b/litellm-rust/crates/cache-azure-blob/tests/cache.rs index b9f50b824b3..3b014f53e82 100644 --- a/litellm-rust/crates/cache-azure-blob/tests/cache.rs +++ b/litellm-rust/crates/cache-azure-blob/tests/cache.rs @@ -128,7 +128,7 @@ fn connect_surfaces_service_failures() { ResponseCacheCodec, runtime.handle().clone(), )); - assert!(matches!(result, Err(Error::Unavailable))); + assert!(matches!(result, Err(Error::Unavailable(_)))); } #[rstest] @@ -263,7 +263,7 @@ fn malformed_blobs_are_invalid_entries(fixture: Fixture, #[case] key: &str, #[ca fixture.service.seed_blob(key, bytes); assert!(matches!( fixture.cache.get_cache(key, &no_ttl()), - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(_)) )); } @@ -378,15 +378,15 @@ fn service_failures_map_to_unavailable(fixture: Fixture) { fixture.service.set_failing(true); assert!(matches!( fixture.cache.get_cache("key", &no_ttl()), - Err(Error::Unavailable) + Err(Error::Unavailable(_)) )); assert!(matches!( fixture.cache.set_cache("key", entry(json!(1)), &no_ttl()), - Err(Error::Unavailable) + Err(Error::Unavailable(_)) )); assert!(matches!( fixture.cache.flush_cache(), - Err(Error::Unavailable) + Err(Error::Unavailable(_)) )); assert!(matches!( fixture.runtime.block_on( @@ -394,7 +394,7 @@ fn service_failures_map_to_unavailable(fixture: Fixture) { .cache .async_set_cache_pipeline(vec![("k".into(), entry(json!(1)))], no_ttl()) ), - Err(Error::Unavailable) + Err(Error::Unavailable(_)) )); } diff --git a/litellm-rust/crates/cache-disk/Cargo.toml b/litellm-rust/crates/cache-disk/Cargo.toml index 825b23899e7..b2f7004910f 100644 --- a/litellm-rust/crates/cache-disk/Cargo.toml +++ b/litellm-rust/crates/cache-disk/Cargo.toml @@ -12,6 +12,7 @@ rand.workspace = true rusqlite = { version = "0.39", features = ["bundled"] } serde-pickle = "1.2" serde_json.workspace = true +thiserror.workspace = true tokio.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache-disk/src/cache.rs b/litellm-rust/crates/cache-disk/src/cache.rs index b8a2c6ed1e4..880a1f86e71 100644 --- a/litellm-rust/crates/cache-disk/src/cache.rs +++ b/litellm-rust/crates/cache-disk/src/cache.rs @@ -64,7 +64,7 @@ impl DiskCache { { tokio::task::spawn_blocking(move || operation(&store)) .await - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? } } @@ -162,7 +162,7 @@ impl BatchCache for DiskCache Ok(BatchEntry::Hit(value)), Ok(None) => Ok(BatchEntry::Miss), - Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(Error::InvalidEntry(_)) => Ok(BatchEntry::Invalid), Err(error) => Err(error), }) .collect() @@ -186,7 +186,7 @@ impl BatchCache for DiskCache match self.decode_stored(value) { Ok(Some(value)) => Ok(BatchEntry::Hit(value)), Ok(None) => Ok(BatchEntry::Miss), - Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(Error::InvalidEntry(_)) => Ok(BatchEntry::Invalid), Err(error) => Err(error), }, }) @@ -272,7 +272,7 @@ fn increment( Ok((stored, ttl.map(|ttl| unix_now() + ttl.as_secs_f64()))) }; store.update(key, unix_now(), &mut apply)?; - result.ok_or(Error::InvalidEntry) + result.ok_or(Error::InvalidEntry(None)) } fn unix_now() -> f64 { diff --git a/litellm-rust/crates/cache-disk/src/python/value.rs b/litellm-rust/crates/cache-disk/src/python/value.rs index 645eba7b757..6e7c2ee0473 100644 --- a/litellm-rust/crates/cache-disk/src/python/value.rs +++ b/litellm-rust/crates/cache-disk/src/python/value.rs @@ -3,8 +3,8 @@ use py_literal::Value; use serde_json::{Map, Number}; pub(crate) fn from_pickle(bytes: &[u8]) -> Result { - let value = serde_pickle::value_from_slice(bytes, Default::default()) - .map_err(|_| Error::InvalidEntry)?; + let value = + serde_pickle::value_from_slice(bytes, Default::default()).map_err(Error::invalid_entry)?; from_pickle_value(value) } @@ -70,7 +70,7 @@ fn from_pickle_hashable(value: serde_pickle::HashableValue) -> Result Result { - value.parse().map_err(|_| Error::InvalidEntry) + value.parse().map_err(Error::invalid_entry) } pub(crate) fn from_json(value: serde_json::Value) -> Value { @@ -101,7 +101,7 @@ pub(crate) fn from_json(value: serde_json::Value) -> Value { pub(crate) fn from_json_text(value: &str) -> Result { serde_json::from_str(value) .map(from_json) - .map_err(|_| Error::InvalidEntry) + .map_err(Error::invalid_entry) } pub(crate) fn is_truthy(value: &Value) -> bool { @@ -131,7 +131,7 @@ pub(crate) fn to_f64(value: &Value) -> Option { } pub(crate) fn to_json(value: &Value) -> Result, Error> { - serde_json::to_vec(&to_json_value(value)?).map_err(|_| Error::InvalidEntry) + serde_json::to_vec(&to_json_value(value)?).map_err(Error::invalid_entry) } fn to_json_value(value: &Value) -> Result { @@ -142,12 +142,12 @@ fn to_json_value(value: &Value) -> Result { value .to_string() .parse::() - .map_err(|_| Error::InvalidEntry)?, + .map_err(Error::invalid_entry)?, ), Value::Float(value) => { - serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry)?) + serde_json::Value::Number(Number::from_f64(*value).ok_or(Error::InvalidEntry(None))?) } - Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry), + Value::Complex(_) | Value::Bytes(_) => return Err(Error::InvalidEntry(None)), Value::String(value) => serde_json::Value::String(value.clone()), Value::Tuple(values) | Value::List(values) | Value::Set(values) => { serde_json::Value::Array( @@ -162,7 +162,7 @@ fn to_json_value(value: &Value) -> Result { .iter() .map(|(key, value)| { let Value::String(key) = key else { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); }; Ok((key.clone(), to_json_value(value)?)) }) diff --git a/litellm-rust/crates/cache-disk/src/sqlite.rs b/litellm-rust/crates/cache-disk/src/sqlite.rs index 24c2be2e1c5..14cbf18b2ff 100644 --- a/litellm-rust/crates/cache-disk/src/sqlite.rs +++ b/litellm-rust/crates/cache-disk/src/sqlite.rs @@ -48,13 +48,13 @@ struct Row { impl DiskcacheSqliteStore { pub fn open(directory: impl AsRef) -> Result { let directory = directory.as_ref().to_path_buf(); - fs::create_dir_all(&directory).map_err(|_| Error::Unavailable)?; - let directory = std::path::absolute(&directory).map_err(|_| Error::Unavailable)?; + fs::create_dir_all(&directory).map_err(Error::unavailable)?; + let directory = std::path::absolute(&directory).map_err(Error::unavailable)?; let database = directory.join("cache.db"); - let connection = Connection::open(database).map_err(|_| Error::Unavailable)?; + let connection = Connection::open(database).map_err(Error::unavailable)?; connection .busy_timeout(std::time::Duration::from_secs(60)) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let mut settings = read_settings(&connection)?; for (key, value) in default_settings() { @@ -74,7 +74,7 @@ impl DiskcacheSqliteStore { value )", ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; for (key, value) in &settings { if !matches!(key.as_str(), "count" | "size" | "hits" | "misses") { connection @@ -82,7 +82,7 @@ impl DiskcacheSqliteStore { "INSERT OR REPLACE INTO Settings VALUES (?, ?)", params![key, value], ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } } for (key, value) in [ @@ -96,7 +96,7 @@ impl DiskcacheSqliteStore { "INSERT OR IGNORE INTO Settings VALUES (?, ?)", params![key, value], ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } connection .execute_batch( @@ -117,7 +117,7 @@ impl DiskcacheSqliteStore { CREATE UNIQUE INDEX IF NOT EXISTS Cache_key_raw ON Cache(key, raw); CREATE INDEX IF NOT EXISTS Cache_expire_time ON Cache(expire_time);", ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let eviction_policy = setting_string(&settings, "eviction_policy") .unwrap_or_else(|| "least-recently-stored".to_string()); @@ -128,23 +128,27 @@ impl DiskcacheSqliteStore { .execute_batch( "CREATE INDEX IF NOT EXISTS Cache_store_time ON Cache(store_time)", ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } "least-recently-used" => { connection .execute_batch( "CREATE INDEX IF NOT EXISTS Cache_access_time ON Cache(access_time)", ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } "least-frequently-used" => { connection .execute_batch( "CREATE INDEX IF NOT EXISTS Cache_access_count ON Cache(access_count)", ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; + } + _ => { + return Err(Error::unavailable(SettingFault::EvictionPolicy( + eviction_policy, + ))); } - _ => return Err(Error::Unavailable), } connection .execute_batch( @@ -170,12 +174,12 @@ impl DiskcacheSqliteStore { UPDATE Settings SET value = value - OLD.size WHERE key = \"size\"; END;", ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let min_file_size = setting_i64(&settings, "disk_min_file_size") .unwrap_or(DEFAULT_DISK_MIN_FILE_SIZE) .try_into() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let size_limit = setting_i64(&settings, "size_limit").unwrap_or(DEFAULT_SIZE_LIMIT); let cull_limit = setting_i64(&settings, "cull_limit").unwrap_or(DEFAULT_CULL_LIMIT); let statistics = setting_i64(&settings, "statistics").unwrap_or_default() != 0; @@ -207,7 +211,7 @@ impl DiskcacheSqliteStore { |row| row.get::<_, Option>(0), ) .optional() - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .flatten() { cleanup.push(old_filename); @@ -221,7 +225,7 @@ impl DiskcacheSqliteStore { |row| row.get::<_, i64>(0), ) .optional() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; if let Some(rowid) = rowid { connection .execute( @@ -230,7 +234,7 @@ impl DiskcacheSqliteStore { WHERE rowid = ?", params![now, expire_time, now, size, mode, filename, value, rowid], ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } else { connection .execute( @@ -240,7 +244,7 @@ impl DiskcacheSqliteStore { ) VALUES (?, 1, ?, ?, ?, 0, NULL, ?, ?, ?, ?)", params![key, now, expire_time, now, size, mode, filename, value], ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } cleanup.extend(self.cull(connection, now)?); Ok(cleanup) @@ -257,13 +261,13 @@ impl DiskcacheSqliteStore { WHERE expire_time IS NOT NULL AND expire_time < ? ORDER BY expire_time LIMIT ?", ) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .query_map(params![now, self.cull_limit], |row| { Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) }) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .collect::, _>>() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; for (_, filename) in &expired { if let Some(filename) = filename { cleanup.push(filename.clone()); @@ -272,7 +276,7 @@ impl DiskcacheSqliteStore { for (rowid, _) in &expired { connection .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid]) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } let remaining = self.cull_limit - i64::try_from(expired.len()).unwrap_or(self.cull_limit); if remaining <= 0 || self.volume(connection)? < self.size_limit { @@ -283,19 +287,23 @@ impl DiskcacheSqliteStore { "least-recently-stored" => "store_time", "least-recently-used" => "access_time", "least-frequently-used" => "access_count", - _ => return Err(Error::Unavailable), + policy => { + return Err(Error::unavailable(SettingFault::EvictionPolicy( + policy.to_owned(), + ))); + } }; let rows = connection .prepare(&format!( "SELECT rowid, filename FROM Cache ORDER BY {order} LIMIT ?" )) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .query_map(params![remaining], |row| { Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) }) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .collect::, _>>() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; for (_, filename) in &rows { if let Some(filename) = filename { cleanup.push(filename.clone()); @@ -304,7 +312,7 @@ impl DiskcacheSqliteStore { for (rowid, _) in rows { connection .execute("DELETE FROM Cache WHERE rowid = ?", params![rowid]) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } Ok(cleanup) } @@ -312,15 +320,15 @@ impl DiskcacheSqliteStore { fn volume(&self, connection: &Connection) -> Result { let page_count: i64 = connection .query_row("PRAGMA page_count", [], |row| row.get(0)) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let page_size: i64 = connection .query_row("PRAGMA page_size", [], |row| row.get(0)) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let size: i64 = connection .query_row("SELECT value FROM Settings WHERE key = 'size'", [], |row| { row.get(0) }) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; Ok(page_count.saturating_mul(page_size).saturating_add(size)) } } @@ -331,13 +339,13 @@ impl DiskStore for DiskcacheSqliteStore { } fn get(&self, key: &str, now: f64) -> Result, Error> { - let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let connection = self.connection.lock()?; let select = "SELECT rowid, expire_time, mode, filename, value FROM Cache WHERE key = ? AND raw = 1 AND (expire_time IS NULL OR expire_time > ?)"; let row = connection .query_row(select, params![key, now], row_from_query) .optional() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; if !self.statistics && !has_get_update(&self.eviction_policy) { return row .map(|row| fetch_row(&self.directory, row)) @@ -348,7 +356,7 @@ impl DiskStore for DiskcacheSqliteStore { let row = connection .query_row(select, params![key, now], row_from_query) .optional() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let Some(row) = row else { if self.statistics { connection @@ -356,7 +364,7 @@ impl DiskStore for DiskcacheSqliteStore { "UPDATE Settings SET value = value + 1 WHERE key = 'misses'", [], ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } return Ok(None); }; @@ -369,14 +377,14 @@ impl DiskStore for DiskcacheSqliteStore { "UPDATE Settings SET value = value + 1 WHERE key = 'hits'", [], ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } else if !hit && self.statistics { connection .execute( "UPDATE Settings SET value = value + 1 WHERE key = 'misses'", [], ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } if has_get_update(&self.eviction_policy) && hit { let update = match self.eviction_policy.as_str() { @@ -384,16 +392,20 @@ impl DiskStore for DiskcacheSqliteStore { "least-frequently-used" => { "UPDATE Cache SET access_count = access_count + 1 WHERE rowid = ?" } - _ => return Err(Error::Unavailable), + policy => { + return Err(Error::unavailable(SettingFault::EvictionPolicy( + policy.to_owned(), + ))); + } }; if self.eviction_policy == "least-recently-used" { connection .execute(update, params![now, rowid]) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } else { connection .execute(update, params![rowid]) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } } value @@ -409,7 +421,7 @@ impl DiskStore for DiskcacheSqliteStore { ) -> Result<(), Error> { let columns = store_value(&self.directory, self.min_file_size, value)?; let new_filename = columns.filename.clone(); - let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let connection = self.connection.lock()?; let result = transactional(&connection, |connection| { self.set_locked(connection, key, columns, expire_time, now) }); @@ -428,7 +440,7 @@ impl DiskStore for DiskcacheSqliteStore { } fn pop(&self, key: &str, now: f64) -> Result, Error> { - let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let connection = self.connection.lock()?; let selected = transactional(&connection, |connection| { let row = connection .query_row( @@ -439,13 +451,13 @@ impl DiskStore for DiskcacheSqliteStore { row_from_query, ) .optional() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let Some(row) = row else { return Ok(None); }; connection .execute("DELETE FROM Cache WHERE rowid = ?", params![row.rowid]) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; Ok(Some(row)) })?; let Some(row) = selected else { @@ -460,7 +472,7 @@ impl DiskStore for DiskcacheSqliteStore { } fn clear(&self) -> Result<(), Error> { - let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let connection = self.connection.lock()?; let mut last_rowid = 0_i64; loop { let batch = transactional(&connection, |connection| { @@ -469,13 +481,13 @@ impl DiskStore for DiskcacheSqliteStore { "SELECT rowid, filename FROM Cache WHERE rowid > ? ORDER BY rowid LIMIT 100", ) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .query_map(params![last_rowid], |row| { Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) }) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .collect::, _>>() - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; if rows.is_empty() { return Ok(rows); } @@ -486,7 +498,7 @@ impl DiskStore for DiskcacheSqliteStore { .join(","); connection .execute(&format!("DELETE FROM Cache WHERE rowid IN ({ids})"), []) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; Ok(rows) })?; if batch.is_empty() { @@ -509,7 +521,7 @@ impl DiskStore for DiskcacheSqliteStore { now: f64, apply: &mut dyn FnMut(Option) -> Result<(StoredValue, Option), Error>, ) -> Result<(), Error> { - let connection = self.connection.lock().map_err(|_| Error::Unavailable)?; + let connection = self.connection.lock()?; let mut created_filename = None; let result = transactional(&connection, |connection| { let current = connection @@ -521,7 +533,7 @@ impl DiskStore for DiskcacheSqliteStore { row_from_query, ) .optional() - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .map(|row| fetch_row(&self.directory, row)) .transpose()? .flatten(); @@ -582,21 +594,35 @@ fn read_settings(connection: &Connection) -> Result, Erro }; statement .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .collect::, _>>() - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) +} + +#[derive(Debug, thiserror::Error)] +enum SettingFault { + #[error("unknown eviction policy {0:?}")] + EvictionPolicy(String), + #[error("setting {0:?} is not an SQLite pragma")] + NotPragma(String), + #[error("setting {0:?} has an unsupported value type")] + PragmaValue(String), } fn apply_pragma(connection: &Connection, key: &str, value: &Value) -> Result<(), Error> { - let pragma = key.strip_prefix("sqlite_").ok_or(Error::Unavailable)?; + let pragma = key + .strip_prefix("sqlite_") + .ok_or_else(|| Error::unavailable(SettingFault::NotPragma(key.to_owned())))?; match value { Value::Integer(value) => connection .pragma_update(None, pragma, value) - .map_err(|_| Error::Unavailable), + .map_err(Error::unavailable), Value::Text(value) => connection .pragma_update(None, pragma, value) - .map_err(|_| Error::Unavailable), - _ => Err(Error::Unavailable), + .map_err(Error::unavailable), + _ => Err(Error::unavailable(SettingFault::PragmaValue( + key.to_owned(), + ))), } } @@ -634,7 +660,7 @@ fn fetch_row(directory: &Path, row: Row) -> Result, Error> { Value::Text(value) => Ok(Some(StoredValue::Text(value))), Value::Integer(value) => Ok(Some(StoredValue::Integer(value))), Value::Real(value) => Ok(Some(StoredValue::Float(value))), - Value::Null => Err(Error::InvalidEntry), + Value::Null => Err(Error::InvalidEntry(None)), }, MODE_BINARY | MODE_PICKLE => { let bytes = match row.value { @@ -645,7 +671,7 @@ fn fetch_row(directory: &Path, row: Row) -> Result, Error> { }; value } - _ => return Err(Error::InvalidEntry), + _ => return Err(Error::InvalidEntry(None)), }; Ok(Some(if row.mode == MODE_BINARY { StoredValue::Bytes(bytes) @@ -663,24 +689,24 @@ fn fetch_row(directory: &Path, row: Row) -> Result, Error> { } Value::Blob(value) => value, Value::Text(value) => value.into_bytes(), - _ => return Err(Error::InvalidEntry), + _ => return Err(Error::InvalidEntry(None)), }; Ok(Some(StoredValue::Text( - String::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?, + String::from_utf8(bytes).map_err(Error::invalid_entry)?, ))) } - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } fn read_file(directory: &Path, filename: Option<&str>) -> Result>, Error> { let Some(filename) = filename else { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); }; match fs::read(directory.join(filename)) { Ok(value) => Ok(Some(value)), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(_) => Err(Error::Unavailable), + Err(error) => Err(Error::unavailable(error)), } } @@ -712,7 +738,7 @@ fn store_value( let bytes = value.into_bytes(); let filename = write_file(directory, &bytes)?; Ok(StoredColumns { - size: i64::try_from(bytes.len()).map_err(|_| Error::Unavailable)?, + size: i64::try_from(bytes.len()).map_err(Error::unavailable)?, mode: MODE_TEXT, filename: Some(filename), value: None, @@ -727,7 +753,7 @@ fn store_value( StoredValue::Bytes(value) => { let filename = write_file(directory, &value)?; Ok(StoredColumns { - size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?, + size: i64::try_from(value.len()).map_err(Error::unavailable)?, mode: MODE_BINARY, filename: Some(filename), value: None, @@ -742,7 +768,7 @@ fn store_value( StoredValue::Pickle(value) => { let filename = write_file(directory, &value)?; Ok(StoredColumns { - size: i64::try_from(value.len()).map_err(|_| Error::Unavailable)?, + size: i64::try_from(value.len()).map_err(Error::unavailable)?, mode: MODE_PICKLE, filename: Some(filename), value: None, @@ -761,14 +787,14 @@ fn write_file(directory: &Path, bytes: &[u8]) -> Result { let filename = format!("{}/{}/{}.val", &hex[..2], &hex[2..4], &hex[4..]); let path = directory.join(&filename); if let Some(parent) = path.parent() { - fs::create_dir_all(parent).map_err(|_| Error::Unavailable)?; + fs::create_dir_all(parent).map_err(Error::unavailable)?; } let mut file = OpenOptions::new() .write(true) .create_new(true) .open(path) - .map_err(|_| Error::Unavailable)?; - file.write_all(bytes).map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; + file.write_all(bytes).map_err(Error::unavailable)?; Ok(filename) } @@ -789,12 +815,12 @@ fn transactional( ) -> Result { connection .execute_batch("BEGIN IMMEDIATE") - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; match operation(connection) { Ok(value) => { connection .execute_batch("COMMIT") - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; Ok(value) } Err(error) => { diff --git a/litellm-rust/crates/cache-disk/tests/python_compat.rs b/litellm-rust/crates/cache-disk/tests/python_compat.rs index 9cbef8573bd..e167b0adb37 100644 --- a/litellm-rust/crates/cache-disk/tests/python_compat.rs +++ b/litellm-rust/crates/cache-disk/tests/python_compat.rs @@ -84,7 +84,7 @@ fn python_read_cases(#[case] row: StoredValue, #[case] expected: ReadExpectation match expected { ReadExpectation::Bytes(expected) => assert_eq!(result.unwrap().unwrap(), expected), ReadExpectation::Miss => assert_eq!(result.unwrap(), None), - ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry))), + ReadExpectation::Invalid => assert!(matches!(result, Err(Error::InvalidEntry(_)))), } } diff --git a/litellm-rust/crates/cache-gcs/Cargo.toml b/litellm-rust/crates/cache-gcs/Cargo.toml index 91630879cbe..dbd8c9ac908 100644 --- a/litellm-rust/crates/cache-gcs/Cargo.toml +++ b/litellm-rust/crates/cache-gcs/Cargo.toml @@ -13,9 +13,11 @@ litellm-auth-types.workspace = true litellm-cache.workspace = true percent-encoding.workspace = true reqwest.workspace = true +thiserror.workspace = true tokio.workspace = true [dev-dependencies] +litellm-error.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-testing.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/cache-gcs/src/cache.rs b/litellm-rust/crates/cache-gcs/src/cache.rs index bad81573cb4..a6b08445bc8 100644 --- a/litellm-rust/crates/cache-gcs/src/cache.rs +++ b/litellm-rust/crates/cache-gcs/src/cache.rs @@ -25,6 +25,14 @@ pub fn key_prefix(gcs_path: Option<&str>) -> String { } } +#[derive(Debug, thiserror::Error)] +#[error("GCS answered {0}")] +struct UnexpectedStatus(reqwest::StatusCode); + +#[derive(Debug, thiserror::Error)] +#[error("GCS cache worker thread panicked")] +struct WorkerPanicked; + #[derive(Clone, Debug)] pub struct GcsConfig { pub bucket_name: String, @@ -114,9 +122,9 @@ impl GcsCache { .body(payload) .send() .await - .map_err(|_| Error::Unavailable)?; + .map_err(|error| Error::unavailable(litellm_http::Error::from(error)))?; if !response.status().is_success() { - return Err(Error::Unavailable); + return Err(Error::unavailable(UnexpectedStatus(response.status()))); } Ok(()) } @@ -135,18 +143,18 @@ impl GcsCache { .header(reqwest::header::CONTENT_TYPE, "application/json") .send() .await - .map_err(|_| Error::Unavailable)?; + .map_err(|error| Error::unavailable(litellm_http::Error::from(error)))?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(None); } if !response.status().is_success() { - return Err(Error::Unavailable); + return Err(Error::unavailable(UnexpectedStatus(response.status()))); } - let body = response.bytes().await.map_err(|_| Error::Unavailable)?; - self.codec - .decode(&body) - .map(Some) - .map_err(|_| Error::InvalidEntry) + let body = response + .bytes() + .await + .map_err(|error| Error::unavailable(litellm_http::Error::from(error)))?; + self.codec.decode(&body).map(Some) } fn run_sync(future: F) -> Result @@ -158,7 +166,7 @@ impl GcsCache { tokio::runtime::Builder::new_current_thread() .enable_all() .build() - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) .and_then(|runtime| runtime.block_on(future)) }; match tokio::runtime::Handle::try_current() { @@ -169,7 +177,7 @@ impl GcsCache { scope .spawn(|| run(future)) .join() - .map_err(|_| Error::Unavailable) + .map_err(|_| Error::unavailable(WorkerPanicked)) .and_then(|result| result) }), Err(_) => run(future), @@ -242,7 +250,7 @@ impl BatchCache for GcsCache { match self.async_get_cache(&key, &context).await { Ok(Some(value)) => Ok(BatchEntry::Hit(value)), Ok(None) => Ok(BatchEntry::Miss), - Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(Error::InvalidEntry(_)) => Ok(BatchEntry::Invalid), Err(error) => Err(error), } } diff --git a/litellm-rust/crates/cache-gcs/src/token.rs b/litellm-rust/crates/cache-gcs/src/token.rs index adb601c276c..079ec44dbce 100644 --- a/litellm-rust/crates/cache-gcs/src/token.rs +++ b/litellm-rust/crates/cache-gcs/src/token.rs @@ -30,7 +30,7 @@ impl TokenSource for GcpTokenSource { self.auth .access_token(&self.config, &|name| std::env::var(name).ok()) .await - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) } } diff --git a/litellm-rust/crates/cache-gcs/tests/cache.rs b/litellm-rust/crates/cache-gcs/tests/cache.rs index 3e92c3fca19..f8dba94ea8e 100644 --- a/litellm-rust/crates/cache-gcs/tests/cache.rs +++ b/litellm-rust/crates/cache-gcs/tests/cache.rs @@ -55,19 +55,27 @@ async fn set_writes_encoded_object_and_headers(#[future(awt)] server: MockServer Ok(Some(json!({"value": "entry"}))) )] #[case::missing("missing", ResponseTemplate::new(404), Ok(None))] -#[case::server_error("server-error", ResponseTemplate::new(500), Err(Error::Unavailable))] -#[case::unauthorized("unauthorized", ResponseTemplate::new(401), Err(Error::Unavailable))] +#[case::server_error( + "server-error", + ResponseTemplate::new(500), + Err("cache is unavailable: GCS answered 500 Internal Server Error") +)] +#[case::unauthorized( + "unauthorized", + ResponseTemplate::new(401), + Err("cache is unavailable: GCS answered 401 Unauthorized") +)] #[case::invalid( "invalid", ResponseTemplate::new(200).set_body_string("not json"), - Err(Error::InvalidEntry) + Err("invalid cache entry: ") )] #[tokio::test] async fn get_maps_statuses_and_decode_failures( #[future(awt)] server: MockServer, #[case] key: &str, #[case] response: ResponseTemplate, - #[case] expected: Result, Error>, + #[case] expected: Result, &str>, ) { Mock::given(method("GET")) .and(path(format!("/storage/v1/b/bucket/o/{key}"))) @@ -76,8 +84,21 @@ async fn get_maps_statuses_and_decode_failures( .mount(&server) .await; let cache = support::cache(&server, None); - assert_eq!(cache.get_cache(key, &context()), expected); - assert_eq!(cache.async_get_cache(key, &context()).await, expected); + let outcome = |result: Result, Error>| { + result.map_err(|error| litellm_error::Report::new(&error).to_string()) + }; + let sync = outcome(cache.get_cache(key, &context())); + let asynchronous = outcome(cache.async_get_cache(key, &context()).await); + match expected { + Ok(value) => { + assert_eq!(sync, Ok(value.clone())); + assert_eq!(asynchronous, Ok(value)); + } + Err(prefix) => { + assert!(sync.unwrap_err().starts_with(prefix)); + assert!(asynchronous.unwrap_err().starts_with(prefix)); + } + } } #[rstest] @@ -257,7 +278,7 @@ struct FailingTokenSource; impl TokenSource for FailingTokenSource { fn bearer_token(&self) -> Pin> + Send + '_>> { - Box::pin(async { Err(Error::Unavailable) }) + Box::pin(async { Err(Error::Unavailable(None)) }) } } @@ -267,14 +288,14 @@ async fn token_source_failure_skips_http(#[future(awt)] server: MockServer) { let cache = support::cache_with_token(&server, None, Arc::new(FailingTokenSource)); assert_eq!( cache.get_cache("key", &context()).unwrap_err(), - Error::Unavailable + Error::Unavailable(None) ); assert_eq!( cache .async_set_cache("key", json!(1), context()) .await .unwrap_err(), - Error::Unavailable + Error::Unavailable(None) ); assert_eq!(server.received_requests().await.unwrap().len(), 0); } diff --git a/litellm-rust/crates/cache-memory/src/cache.rs b/litellm-rust/crates/cache-memory/src/cache.rs index 74476559cbe..73fbf14b07a 100644 --- a/litellm-rust/crates/cache-memory/src/cache.rs +++ b/litellm-rust/crates/cache-memory/src/cache.rs @@ -2,7 +2,7 @@ use std::{ cmp::Reverse, collections::{BinaryHeap, HashMap, HashSet}, hash::Hash, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, MutexGuard}, time::{Duration, SystemTime, UNIX_EPOCH}, }; @@ -84,6 +84,10 @@ impl InMemoryCache { } } + fn lock(&self) -> Result>, Error> { + Ok(self.state.lock()?) + } + pub fn set_cache( &self, key: impl Into, @@ -94,13 +98,13 @@ impl InMemoryCache { return Ok(CacheWrite::Disabled); } let now = (self.now)(); - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; self.store(&mut state, key.into(), value, ttl, now) } pub fn get_cache(&self, key: &str) -> Result, Error> { let now = (self.now)(); - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; if state .expirations .get(key) @@ -124,7 +128,7 @@ impl InMemoryCache { /// fits. pub fn evict_cache(&self) -> Result<(), Error> { let now = (self.now)(); - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; Self::evict(&mut state, self.max_size_in_memory, now, None); Ok(()) } @@ -132,7 +136,7 @@ impl InMemoryCache { /// `evict_element_if_expired`: `true` when `key` had expired and was removed. pub fn evict_element_if_expired(&self, key: &str) -> Result { let now = (self.now)(); - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; let expired = state .expirations .get(key) @@ -153,12 +157,7 @@ impl InMemoryCache { /// The number of stored entries, expired ones included until they are evicted. pub fn len(&self) -> Result { - Ok(self - .state - .lock() - .map_err(|_| Error::Unavailable)? - .values - .len()) + Ok(self.lock()?.values.len()) } pub fn is_empty(&self) -> Result { @@ -167,12 +166,7 @@ impl InMemoryCache { /// Entries in the expiration heap, stale ones included; bounded by eviction. pub fn expiration_heap_len(&self) -> Result { - Ok(self - .state - .lock() - .map_err(|_| Error::Unavailable)? - .expiration_heap - .len()) + Ok(self.lock()?.expiration_heap.len()) } pub fn max_size_in_memory(&self) -> usize { @@ -184,13 +178,7 @@ impl InMemoryCache { } pub fn expires_at(&self, key: &str) -> Result, Error> { - Ok(self - .state - .lock() - .map_err(|_| Error::Unavailable)? - .expirations - .get(key) - .copied()) + Ok(self.lock()?.expirations.get(key).copied()) } pub async fn async_get_ttl(&self, key: &str) -> Result, Error> { @@ -198,7 +186,7 @@ impl InMemoryCache { } pub async fn async_get_oldest_n_keys(&self, count: usize) -> Result, Error> { - let state = self.state.lock().map_err(|_| Error::Unavailable)?; + let state = self.lock()?; let mut expirations = state .expirations .iter() @@ -213,13 +201,13 @@ impl InMemoryCache { } pub fn delete_cache(&self, key: &str) -> Result<(), Error> { - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; Self::remove(&mut state, key); Ok(()) } pub fn flush_cache(&self) -> Result<(), Error> { - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; state.values.clear(); state.expirations.clear(); state.expiration_heap.clear(); @@ -316,7 +304,7 @@ where return Ok(candidate); } let now = (self.now)(); - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; Self::evict(&mut state, self.max_size_in_memory, now, Some(key)); let existing = state .values @@ -351,7 +339,7 @@ impl CounterCache for InMemoryCache { return Ok(amount); } let now = (self.now)(); - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; let value = Self::live(&mut state, key, now).unwrap_or_default() + amount; self.store(&mut state, key.into(), value, self.get_ttl(&context), now)?; Ok(value) @@ -424,7 +412,7 @@ where return Ok(values); } let now = (self.now)(); - let mut state = self.state.lock().map_err(|_| Error::Unavailable)?; + let mut state = self.lock()?; let mut stored = Self::live(&mut state, key, now).unwrap_or_default(); stored.extend(values.iter().cloned()); self.store(&mut state, key.into(), stored, ttl, now)?; diff --git a/litellm-rust/crates/cache-memory/tests/cache.rs b/litellm-rust/crates/cache-memory/tests/cache.rs index ce7d6ac8a98..e6c74e5ccda 100644 --- a/litellm-rust/crates/cache-memory/tests/cache.rs +++ b/litellm-rust/crates/cache-memory/tests/cache.rs @@ -52,7 +52,7 @@ fn measured(capacity: usize) -> InMemoryCache { Some(4), Some(Arc::new(|value: &String| { if value.is_empty() { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } Ok(value.len()) })), @@ -275,7 +275,7 @@ fn evict_element_if_expired_reports_removal(clock: Clock) { #[case::fits("ok", Ok(true))] #[case::at_limit("four", Ok(true))] #[case::too_large("oversized", Ok(false))] -#[case::measure_error("", Err(Error::InvalidEntry))] +#[case::measure_error("", Err(Error::InvalidEntry(None)))] fn check_value_size_applies_the_entry_limit( #[case] value: &str, #[case] expected: Result, @@ -310,7 +310,7 @@ fn disabled_size_limited_and_validated_writes_are_observable() { assert_eq!(cache.get_cache("small").unwrap(), Some("ok".into())); assert_eq!( cache.set_cache("invalid", String::new(), None), - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); assert_eq!(cache.get_cache("invalid").unwrap(), None); cache.delete_cache("small").unwrap(); diff --git a/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml index a44bef0a731..8a4caa8def7 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-qdrant-semantic/Cargo.toml @@ -18,6 +18,7 @@ tokio.workspace = true uuid.workspace = true [dev-dependencies] +litellm-error.workspace = true litellm-http = { workspace = true, features = ["test-support"] } futures-executor = "0.3" litellm-cache-testing.workspace = true diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/cache.rs b/litellm-rust/crates/cache-qdrant-semantic/src/cache.rs index a140e0af174..9c93cc45386 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/src/cache.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/src/cache.rs @@ -36,7 +36,7 @@ impl QdrantSemanticCache { let exists = client .collection_exists(config.collection_name.clone()) .await - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; if !exists { client .create_collection( @@ -48,7 +48,7 @@ impl QdrantSemanticCache { .quantization_config(quantization(&config.quantization)), ) .await - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } let _ = client .create_field_index(CreateFieldIndexCollectionBuilder::new( @@ -99,13 +99,13 @@ impl QdrantSemanticCache { .async_embed(&prompt, context.metadata.as_ref()) .await?; let response = - String::from_utf8(self.codec.encode(&value)?).map_err(|_| Error::InvalidEntry)?; + String::from_utf8(self.codec.encode(&value)?).map_err(Error::invalid_entry)?; let payload = Payload::try_from(json!({ "litellm_cache_key": key, "text": prompt, "response": response, })) - .map_err(|_| Error::InvalidEntry)?; + .map_err(Error::invalid_entry)?; self.client .upsert_points( UpsertPointsBuilder::new( @@ -119,7 +119,7 @@ impl QdrantSemanticCache { .wait(true), ) .await - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; Ok(()) } @@ -152,7 +152,7 @@ impl QdrantSemanticCache { ), ) .await - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let Some(point) = result.result.into_iter().next() else { return Ok(SemanticLookup::miss(Some(0.0))); }; @@ -170,7 +170,7 @@ impl QdrantSemanticCache { let response = payload .get("response") .and_then(Value::as_str) - .ok_or(Error::InvalidEntry)?; + .ok_or(Error::InvalidEntry(None))?; Ok(SemanticLookup { value: Some(self.codec.decode(response.as_bytes())?), similarity: Some(similarity), diff --git a/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs index b7fbcd9b02d..bad780bc57f 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/src/embedder.rs @@ -55,10 +55,9 @@ impl Embedder for OpenAiEmbedder { } .send() .await - .map_err(|_| Error::Unavailable)? - .error_for_status() - .map_err(|_| Error::Unavailable)?; - let body: Value = response.json().await.map_err(|_| Error::Unavailable)?; + .and_then(reqwest::Response::error_for_status) + .map_err(http_error)?; + let body: Value = response.json().await.map_err(http_error)?; body.get("data") .and_then(Value::as_array) .and_then(|data| data.first()) @@ -70,6 +69,14 @@ impl Embedder for OpenAiEmbedder { .map(|value| value.as_f64().map(|value| value as f32)) .collect::>>() }) - .ok_or(Error::Unavailable) + .ok_or(Error::unavailable(MissingEmbedding)) } } + +fn http_error(error: reqwest::Error) -> Error { + Error::unavailable(litellm_http::Error::from(error)) +} + +#[derive(Debug, thiserror::Error)] +#[error("embeddings response has no embedding")] +struct MissingEmbedding; diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs index 71fc986616e..42261d21524 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/embedder.rs @@ -134,13 +134,19 @@ async fn posts_embeddings_request_and_parses_vector() { } #[rstest] -#[case::error_status("500 Internal Server Error", "{}", 0, None, Err(Error::Unavailable))] +#[case::error_status( + "500 Internal Server Error", + "{}", + 0, + None, + Err(Error::Unavailable(None)) +)] #[case::timed_out( "200 OK", r#"{"data":[{"embedding":[0.1,0.2]}]}"#, 500, Some(Duration::from_millis(200)), - Err(Error::Unavailable) + Err(Error::Unavailable(None)) )] #[case::within_timeout( "200 OK", @@ -149,7 +155,7 @@ async fn posts_embeddings_request_and_parses_vector() { Some(Duration::from_secs(1)), Ok(vec![0.1, 0.2]) )] -#[case::missing_embedding("200 OK", r#"{"data":[]}"#, 0, None, Err(Error::Unavailable))] +#[case::missing_embedding("200 OK", r#"{"data":[]}"#, 0, None, Err(Error::Unavailable(None)))] #[tokio::test] async fn status_timeout_and_body_errors_are_unavailable( #[case] status: &str, @@ -164,7 +170,14 @@ async fn status_timeout_and_body_errors_are_unavailable( litellm_http::Client::for_test(), config(server.base_url(), timeout), ); - assert_eq!(embedder.async_embed("hello", None).await, expected); + let result = embedder.async_embed("hello", None).await; + if let Err(error) = &result { + let report = litellm_error::Report::new(error).to_string(); + assert!(!report.contains(&server.base_url()), "{report}"); + } + let kind = + |result: Result, Error>| result.map_err(|error| std::mem::discriminant(&error)); + assert_eq!(kind(result), kind(expected)); } #[rstest] diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index fe25c503a36..b0764b4470b 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -53,7 +53,10 @@ impl Embedder for FixedEmbedder { .lock() .unwrap() .push((input.to_owned(), metadata.cloned())); - self.vectors.get(input).cloned().ok_or(Error::Unavailable) + self.vectors + .get(input) + .cloned() + .ok_or(Error::Unavailable(None)) } } @@ -396,9 +399,9 @@ async fn lookup_reports_python_semantic_similarity( #[rstest] #[case::codec_decodes_the_payload(Some(json!("{\"a\":1}")), Ok(Some(json!({"a": 1}))))] -#[case::undecodable_response(Some(json!("not json")), Err(Error::InvalidEntry))] -#[case::non_string_response(Some(json!(1)), Err(Error::InvalidEntry))] -#[case::missing_response(None, Err(Error::InvalidEntry))] +#[case::undecodable_response(Some(json!("not json")), Err(Error::InvalidEntry(None)))] +#[case::non_string_response(Some(json!(1)), Err(Error::InvalidEntry(None)))] +#[case::missing_response(None, Err(Error::InvalidEntry(None)))] #[tokio::test(flavor = "multi_thread")] async fn stored_responses_go_through_the_codec( #[case] response: Option, @@ -419,9 +422,12 @@ async fn stored_responses_go_through_the_codec( .into(), }); + let kind = |result: Result, Error>| { + result.map_err(|error| std::mem::discriminant(&error)) + }; assert_eq!( - cache.async_get_cache("key", &context("hello")).await, - expected + kind(cache.async_get_cache("key", &context("hello")).await), + kind(expected) ); server.stop(); } @@ -434,7 +440,7 @@ async fn embedding_failures_propagate() { assert_eq!( cache.async_get_cache("key", &context("unknown")).await, - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); server.stop(); } @@ -493,10 +499,10 @@ async fn stopped_qdrant_server_maps_to_unavailable() { let cache = connect(&server, [("hello", vec![1.0, 0.0])]).await; server.stop(); tokio::time::sleep(Duration::from_millis(50)).await; - assert_eq!( + assert!(matches!( cache.async_get_cache("key", &context("hello")).await, - Err(Error::Unavailable) - ); + Err(Error::Unavailable(Some(_))) + )); } /// `_payload_matches_cache_key` compares `str(cached_key) == str(key)`, so a point whose stored diff --git a/litellm-rust/crates/cache-redis-semantic/src/cache.rs b/litellm-rust/crates/cache-redis-semantic/src/cache.rs index 02feb3cbdf4..42fc6ae585f 100644 --- a/litellm-rust/crates/cache-redis-semantic/src/cache.rs +++ b/litellm-rust/crates/cache-redis-semantic/src/cache.rs @@ -63,13 +63,13 @@ impl Inner { .arg(CACHE_KEY_FIELD) .arg(tag) .query::<()>(connection) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; if let Some(ttl) = ttl { redis::cmd("EXPIRE") .arg(&hash_key) .arg(ttl_seconds(ttl)) .query::<()>(connection) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } Ok(()) } @@ -111,7 +111,7 @@ impl Inner { .arg("vector") .arg(vector_buffer(vector)) .query::(connection) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let Some(fields) = first_document(&result) else { return Ok(SemanticLookup::miss(Some(0.0))); }; diff --git a/litellm-rust/crates/cache-redis-semantic/src/index.rs b/litellm-rust/crates/cache-redis-semantic/src/index.rs index c141e47003a..23401938a0e 100644 --- a/litellm-rust/crates/cache-redis-semantic/src/index.rs +++ b/litellm-rust/crates/cache-redis-semantic/src/index.rs @@ -40,10 +40,10 @@ impl Index { Some(false) => self.isolated(connection, dims)?, None => match create_index(connection, &self.name, dims) { Ok(()) => self.name.clone(), - Err(_) => match index_compatible(connection, &self.name, dims)? { + Err(error) => match index_compatible(connection, &self.name, dims)? { Some(true) => self.name.clone(), Some(false) => self.isolated(connection, dims)?, - None => return Err(Error::Unavailable), + None => return Err(error), }, }, }; @@ -59,7 +59,7 @@ impl Index { redis::cmd("FT.DROPINDEX") .arg(&name) .query::<()>(connection) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; create_index(connection, &name, dims)?; Ok(name) } @@ -109,7 +109,7 @@ fn create_index(connection: &mut ConnectionRef<'_>, name: &str, dims: usize) -> .arg("SEPARATOR") .arg(",") .query::<()>(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } fn index_compatible( @@ -123,14 +123,18 @@ fn index_compatible( { Ok(info) => info, Err(error) if unknown_index(&error) => return Ok(None), - Err(_) => return Err(Error::Unavailable), + Err(error) => return Err(Error::unavailable(error)), }; Ok(Some(schema_compatible(&info, dims))) } fn unknown_index(error: &redis::RedisError) -> bool { - let message = error.to_string().to_lowercase(); - message.contains("unknown") && message.contains("index") + error + .code() + .is_some_and(|code| code.eq_ignore_ascii_case("unknown")) + && error + .detail() + .is_some_and(|detail| detail.to_ascii_lowercase().starts_with("index")) } struct Attribute { diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs index fa4ac00e767..cc42f4c3924 100644 --- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs @@ -79,7 +79,7 @@ fn s(value: &str) -> redis::Value { } fn unknown_index_error() -> redis::RedisError { - redis::RedisError::from((redis::ErrorKind::Extension, "Unknown index name")) + support::server_error("Unknown index name") } fn info_missing(index: &str) -> MockCmd { @@ -320,7 +320,7 @@ fn store_writes_the_redisvl_hash( #[case::below_distance_threshold("key1", "0.05", None, Ok(Some(entry())))] #[case::above_distance_threshold("key1", "0.5", None, Ok(None))] #[case::other_cache_key("other", "0.05", None, Ok(None))] -#[case::malformed_response("key1", "0.05", Some(b"not json!".as_slice()), Err(Error::InvalidEntry))] +#[case::malformed_response("key1", "0.05", Some(b"not json!".as_slice()), Err(Error::InvalidEntry(None)))] fn lookup_applies_threshold_scope_and_codec( #[case] stored_tag: &str, #[case] distance: &str, @@ -343,7 +343,10 @@ fn lookup_applies_threshold_scope_and_codec( FakeEmbedder::new(&[]), ); - assert_eq!(cache.get_cache("key1", &context), expected); + let kind = |result: Result, Error>| { + result.map_err(|error| std::mem::discriminant(&error)) + }; + assert_eq!(kind(cache.get_cache("key1", &context)), kind(expected)); } #[rstest] @@ -451,10 +454,7 @@ fn create_index_race_rechecks_schema_and_stores(entry: Value, context: SemanticC info_missing(INDEX), MockCmd::new( create_index_command(INDEX, 3), - Err::<&str, _>(redis::RedisError::from(( - redis::ErrorKind::Extension, - "Index already exists", - ))), + Err::<&str, _>(support::server_error("Index already exists")), ), info(INDEX, compatible_info(3)), hset(INDEX, PROMPT, "key1", &VECTOR, &entry), @@ -673,10 +673,10 @@ fn shared_base_index_across_dimensions_replaces_the_isolated_index(entry: Value) .set_cache("key1", entry.clone(), &context()) .unwrap(); - assert_eq!( - worker_b.get_cache("key1", &context()).unwrap_err(), - Error::Unavailable - ); + assert!(matches!( + worker_b.get_cache("key1", &context()), + Err(Error::Unavailable(Some(_))) + )); } #[fixture] @@ -732,10 +732,10 @@ fn live_shared_index_is_replaced_across_dimensions(redis_stack_url: Option { let name = text(1); if state.indexes.contains_key(&name) { - return Err(error("Index already exists")); + return Err(server_error("Index already exists")); } let position = |token: &str| args.iter().position(|arg| arg == token.as_bytes()); let prefix = args[position("PREFIX").unwrap() + 2].clone(); @@ -94,7 +94,7 @@ impl FakeSearch { let index = state .indexes .get(&text(1)) - .ok_or_else(|| error("Unknown index name"))?; + .ok_or_else(|| server_error("Unknown index name"))?; Ok(index_info(index)) } "FT.DROPINDEX" => { @@ -118,7 +118,7 @@ impl FakeSearch { let index = state .indexes .get(&text(1)) - .ok_or_else(|| error("no such index"))?; + .ok_or_else(|| server_error("no such index"))?; let query = text(2); let tag = query_tag(&query); let params = args.iter().position(|arg| arg == b"PARAMS").unwrap(); @@ -155,7 +155,7 @@ impl FakeSearch { ])) } "PING" => Ok(redis::Value::SimpleString("PONG".into())), - _ => Err(error("unsupported command")), + _ => Err(server_error("unsupported command")), } } } @@ -192,8 +192,11 @@ impl redis::ConnectionLike for FakeSearch { } } -fn error(message: &'static str) -> redis::RedisError { - redis::RedisError::from((redis::ErrorKind::Extension, message)) +pub fn server_error(reply: &str) -> redis::RedisError { + redis::parse_redis_value(format!("-{reply}\r\n").as_bytes()) + .expect("error reply") + .extract_error() + .expect_err("server error") } fn bulk(bytes: &[u8]) -> redis::Value { diff --git a/litellm-rust/crates/cache-redis/Cargo.toml b/litellm-rust/crates/cache-redis/Cargo.toml index a234286a338..9b66c7ff05d 100644 --- a/litellm-rust/crates/cache-redis/Cargo.toml +++ b/litellm-rust/crates/cache-redis/Cargo.toml @@ -7,8 +7,10 @@ repository.workspace = true [dependencies] litellm-cache.workspace = true +litellm-error.workspace = true redis = { version = "1.7.0", features = ["cluster", "tls-rustls"] } r2d2 = "0.8.10" +thiserror.workspace = true tokio.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache-redis/src/cache.rs b/litellm-rust/crates/cache-redis/src/cache.rs index ebcf0b6916b..b873a2c0282 100644 --- a/litellm-rust/crates/cache-redis/src/cache.rs +++ b/litellm-rust/crates/cache-redis/src/cache.rs @@ -141,7 +141,7 @@ where redis::Value::Nil => Ok(None), redis::Value::BulkString(bytes) => self.codec.decode(&bytes).map(Some), redis::Value::SimpleString(text) => self.codec.decode(text.as_bytes()).map(Some), - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } @@ -152,7 +152,7 @@ where match self.decode_response(value) { Ok(Some(value)) => Ok(BatchEntry::Hit(value)), Ok(None) => Ok(BatchEntry::Miss), - Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(Error::InvalidEntry(_)) => Ok(BatchEntry::Invalid), Err(error) => Err(error), } } diff --git a/litellm-rust/crates/cache-redis/src/claim.rs b/litellm-rust/crates/cache-redis/src/claim.rs index e4cbe8c8d5c..c57a1b03395 100644 --- a/litellm-rust/crates/cache-redis/src/claim.rs +++ b/litellm-rust/crates/cache-redis/src/claim.rs @@ -17,7 +17,7 @@ fn stored_bytes(value: redis::Value) -> Result>, Error> { redis::Value::Nil => Ok(None), redis::Value::BulkString(bytes) => Ok(Some(bytes)), redis::Value::SimpleString(text) => Ok(Some(text.into_bytes())), - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } @@ -37,13 +37,13 @@ where { let payload = codec.encode(&candidate)?; if payload.is_empty() { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } for _ in 0..CLAIM_ATTEMPTS { let current = stored_bytes( connection .get::<_, redis::Value>(key) - .map_err(|_| Error::Unavailable)?, + .map_err(Error::unavailable)?, )? .filter(|bytes| !bytes.is_empty()); let existing = current @@ -63,14 +63,18 @@ where .arg(write) .arg(u8::from(refresh)) .query::(connection) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; if applied { return Ok(existing.unwrap_or(candidate)); } } - Err(Error::Unavailable) + Err(Error::unavailable(ClaimContended)) } +#[derive(Debug, thiserror::Error)] +#[error("claim kept losing the race after {CLAIM_ATTEMPTS} attempts")] +struct ClaimContended; + impl ClaimCache for RedisCache where S: CacheCodec + Clone + 'static, diff --git a/litellm-rust/crates/cache-redis/src/connection.rs b/litellm-rust/crates/cache-redis/src/connection.rs index 2f58e2a9b80..99befa16ffa 100644 --- a/litellm-rust/crates/cache-redis/src/connection.rs +++ b/litellm-rust/crates/cache-redis/src/connection.rs @@ -36,19 +36,19 @@ where ) -> Result { match self { Self::Pool(pool) => { - let mut pooled = pool.get().map_err(|_| Error::Unavailable)?; + let mut pooled = pool.get().map_err(Error::unavailable)?; let result = operation(&mut ConnectionRef::Node(&mut pooled.connection)); - pooled.failed = matches!(result, Err(Error::Unavailable)); + pooled.failed = matches!(result, Err(Error::Unavailable(_))); result } Self::Cluster(pool) => { - let mut pooled = pool.get().map_err(|_| Error::Unavailable)?; + let mut pooled = pool.get().map_err(Error::unavailable)?; let result = operation(&mut ConnectionRef::Cluster(&mut pooled.connection)); - pooled.failed = matches!(result, Err(Error::Unavailable)); + pooled.failed = matches!(result, Err(Error::Unavailable(_))); result } Self::Fixed(connection) => { - let mut connection = connection.lock().map_err(|_| Error::Unavailable)?; + let mut connection = connection.lock()?; operation(&mut ConnectionRef::Node(&mut *connection)) } } @@ -61,7 +61,7 @@ where { tokio::task::spawn_blocking(move || connections.execute(operation)) .await - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? } pub fn fixed(connection: C) -> Self { @@ -95,7 +95,7 @@ fn pool(manager: M) -> Result, Error> { .connection_timeout(REDIS_TIMEOUT) .test_on_check_out(false) .build(manager) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } fn close_idle(pool: &r2d2::Pool) @@ -123,7 +123,7 @@ impl ConnectionManager { fn open(url: &str) -> Result { redis::Client::open(url) .map(Self) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } } @@ -151,14 +151,26 @@ impl r2d2::ManageConnection for ConnectionManager { } } +#[derive(Debug, thiserror::Error)] +enum TopologyFault { + #[error("Redis Cluster needs at least one startup node")] + NoStartupNodes, + #[error("Redis Cluster nodes need a TCP address")] + NonTcpAddress, + #[error("Redis Cluster answered a fan-out command without a per-node map")] + FanOutReply, + #[error("scan target does not match the connection topology")] + ScanTarget, +} + pub(crate) struct ClusterConnectionManager(ClusterClient); impl ClusterConnectionManager { fn open(url: &str, startup_nodes: &[RedisNode]) -> Result { if startup_nodes.is_empty() { - return Err(Error::Unavailable); + return Err(Error::unavailable(TopologyFault::NoStartupNodes)); } - let info = url.into_connection_info().map_err(|_| Error::Unavailable)?; + let info = url.into_connection_info().map_err(Error::unavailable)?; let nodes = startup_nodes .iter() .map(|node| node_info(&info, node)) @@ -168,7 +180,7 @@ impl ClusterConnectionManager { .response_timeout(REDIS_TIMEOUT) .build() .map(Self) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } } @@ -185,7 +197,7 @@ fn node_info(info: &ConnectionInfo, node: &RedisNode) -> Result return Err(Error::Unavailable), + _ => return Err(Error::unavailable(TopologyFault::NonTcpAddress)), }; Ok(info.clone().set_addr(addr)) } @@ -298,7 +310,7 @@ impl ConnectionRef<'_> { .and_then(|replies| pipeline.query(&mut Replies(Some(replies)))) } } - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } pub(crate) fn scan( @@ -311,7 +323,7 @@ impl ConnectionRef<'_> { Self::Node(connection) => { let page = scan_command(0, pattern, count) .query::(*connection) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; vec![(None, page)] } Self::Cluster(connection) => connection @@ -322,7 +334,7 @@ impl ConnectionRef<'_> { Some(ResponsePolicy::Special), )), ) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) .and_then(primary_pages)? .into_iter() .map(|(node, page)| (Some(node), page)) @@ -362,7 +374,7 @@ impl ConnectionRef<'_> { pub(crate) fn node_text(&mut self, command: &redis::Cmd) -> Result { match self { - Self::Node(connection) => command.query(*connection).map_err(|_| Error::Unavailable), + Self::Node(connection) => command.query(*connection).map_err(Error::unavailable), Self::Cluster(connection) => { let value = connection .route_command( @@ -372,18 +384,16 @@ impl ConnectionRef<'_> { Some(ResponsePolicy::Special), )), ) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let redis::Value::Map(entries) = value else { - return Err(Error::Unavailable); + return Err(Error::unavailable(TopologyFault::FanOutReply)); }; let mut replies = entries .into_iter() .map(|(node, reply)| { Ok(( - redis::from_redis_value::(node) - .map_err(|_| Error::Unavailable)?, - redis::from_redis_value::(reply) - .map_err(|_| Error::Unavailable)?, + redis::from_redis_value::(node).map_err(Error::unavailable)?, + redis::from_redis_value::(reply).map_err(Error::unavailable)?, )) }) .collect::, Error>>()?; @@ -400,7 +410,7 @@ impl ConnectionRef<'_> { pub(crate) fn flushall(&mut self) -> Result<(), Error> { let command = redis::cmd("FLUSHALL"); match self { - Self::Node(connection) => command.query(*connection).map_err(|_| Error::Unavailable), + Self::Node(connection) => command.query(*connection).map_err(Error::unavailable), Self::Cluster(connection) => connection .route_command( &command, @@ -410,7 +420,7 @@ impl ConnectionRef<'_> { )), ) .map(|_| ()) - .map_err(|_| Error::Unavailable), + .map_err(Error::unavailable), } } @@ -424,7 +434,7 @@ impl ConnectionRef<'_> { let command = scan_command(cursor, pattern, count); match (self, node) { (Self::Node(connection), None) => { - command.query(*connection).map_err(|_| Error::Unavailable) + command.query(*connection).map_err(Error::unavailable) } (Self::Cluster(connection), Some(node)) => connection .route_command( @@ -434,9 +444,9 @@ impl ConnectionRef<'_> { port: node.port(), }), ) - .map_err(|_| Error::Unavailable) - .and_then(|value| redis::from_redis_value(value).map_err(|_| Error::Unavailable)), - _ => Err(Error::Unavailable), + .map_err(Error::unavailable) + .and_then(|value| redis::from_redis_value(value).map_err(Error::unavailable)), + _ => Err(Error::unavailable(TopologyFault::ScanTarget)), } } } @@ -445,14 +455,14 @@ type ScanPage = (u64, Vec); fn primary_pages(value: redis::Value) -> Result, Error> { let redis::Value::Map(entries) = value else { - return Err(Error::Unavailable); + return Err(Error::unavailable(TopologyFault::FanOutReply)); }; entries .into_iter() .map(|(node, page)| { - let node = redis::from_redis_value::(node).map_err(|_| Error::Unavailable)?; - let node = NodeAddress::try_from(node.as_str()).map_err(|_| Error::Unavailable)?; - let page = redis::from_redis_value::(page).map_err(|_| Error::Unavailable)?; + let node = redis::from_redis_value::(node).map_err(Error::unavailable)?; + let node = NodeAddress::try_from(node.as_str()).map_err(Error::unavailable)?; + let page = redis::from_redis_value::(page).map_err(Error::unavailable)?; Ok((node, page)) }) .collect() diff --git a/litellm-rust/crates/cache-redis/src/counter.rs b/litellm-rust/crates/cache-redis/src/counter.rs index adcd597b2f4..a4beb9f5712 100644 --- a/litellm-rust/crates/cache-redis/src/counter.rs +++ b/litellm-rust/crates/cache-redis/src/counter.rs @@ -100,7 +100,7 @@ fn increment( .arg(amount) .arg(ttl) .query(connection) - .map_err(|_| Error::Unavailable); + .map_err(Error::unavailable); } connection .query_pipeline( @@ -140,7 +140,7 @@ where } fn count(value: redis::Value) -> Result, Error> { - redis::from_redis_value(value).map_err(|_| Error::InvalidEntry) + redis::from_redis_value(value).map_err(Error::invalid_entry) } impl BoundedCounterCache for RedisCache @@ -182,7 +182,7 @@ where .arg(value) .arg(ttl) .query(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) .await } @@ -201,5 +201,5 @@ fn increment_with_floor( .arg(amount) .arg(ttl) .query(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } diff --git a/litellm-rust/crates/cache-redis/src/keys.rs b/litellm-rust/crates/cache-redis/src/keys.rs index b9d7bb20ff3..2e8a9e60e65 100644 --- a/litellm-rust/crates/cache-redis/src/keys.rs +++ b/litellm-rust/crates/cache-redis/src/keys.rs @@ -16,7 +16,7 @@ where redis::cmd("TTL") .arg(key) .query::(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) .await?; Ok(u64::try_from(ttl).ok().map(Duration::from_secs)) @@ -36,7 +36,7 @@ where .arg(key) .arg(ttl) .query(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) .await } diff --git a/litellm-rust/crates/cache-redis/src/lifecycle.rs b/litellm-rust/crates/cache-redis/src/lifecycle.rs index 6ab8c31509e..54fa8ecc3bf 100644 --- a/litellm-rust/crates/cache-redis/src/lifecycle.rs +++ b/litellm-rust/crates/cache-redis/src/lifecycle.rs @@ -11,11 +11,11 @@ where C: redis::ConnectionLike + Send + 'static, { fn sync_ping(&self) -> Result { - self.execute(|connection| connection.ping().map_err(|_| Error::Unavailable)) + self.execute(|connection| connection.ping().map_err(Error::unavailable)) } async fn ping(&self) -> Result { - self.run(|connection| connection.ping().map_err(|_| Error::Unavailable)) + self.run(|connection| connection.ping().map_err(Error::unavailable)) .await } } @@ -33,9 +33,10 @@ where RedisTopology::Cluster { .. } => "Redis Cluster", }; let ping = self - .run(|connection| Ok(connection.ping().map_err(|error| error.to_string()))) + .run(|connection| Ok(connection.ping().map_err(Error::unavailable))) .await - .unwrap_or_else(|error| Err(error.to_string())); + .and_then(|ping| ping) + .map_err(|error| litellm_error::Report::new(&error).to_string()); Ok(match ping { Ok(true) => CacheConnectionResult { status: CacheConnectionStatus::Success, diff --git a/litellm-rust/crates/cache-redis/src/queue.rs b/litellm-rust/crates/cache-redis/src/queue.rs index 623bce23640..d55e4f49187 100644 --- a/litellm-rust/crates/cache-redis/src/queue.rs +++ b/litellm-rust/crates/cache-redis/src/queue.rs @@ -29,7 +29,7 @@ where ttl: Option, ) -> Result { if values.is_empty() { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } let key = self.namespaced_key(key); let ttl = self.ttl_or_default(ttl); @@ -58,7 +58,7 @@ where async fn async_rpush(&self, key: &str, values: Vec) -> Result { if values.is_empty() { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } let key = self.namespaced_key(key); self.run(move |connection| { @@ -66,7 +66,7 @@ where .arg(key) .arg(values) .query(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) .await } @@ -78,7 +78,7 @@ where max_len: usize, ) -> Result { if values.is_empty() { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } let key = self.namespaced_key(key); let start = i64::try_from(max_len).map_or(i64::MIN, |max_len| -max_len); @@ -108,7 +108,7 @@ where let mut pipeline = redis::pipe(); for operation in operations { if operation.values.is_empty() { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } pipeline .cmd("RPUSH") @@ -130,7 +130,7 @@ where .run(move |connection| { command .query::(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) .await?; lpop_result(value, count.is_some()) @@ -182,7 +182,7 @@ fn redis_bytes(value: redis::Value) -> Result, Error> { match value { redis::Value::BulkString(bytes) => Ok(bytes), redis::Value::SimpleString(text) => Ok(text.into_bytes()), - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } @@ -221,6 +221,6 @@ fn lpop_result(value: redis::Value, multiple: bool) -> Result, _>>() .map(RedisLpopResult::Values), value if !multiple => redis_bytes(value).map(RedisLpopResult::Value), - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } diff --git a/litellm-rust/crates/cache-redis/src/script.rs b/litellm-rust/crates/cache-redis/src/script.rs index e8a16bcc2b5..09987437ed4 100644 --- a/litellm-rust/crates/cache-redis/src/script.rs +++ b/litellm-rust/crates/cache-redis/src/script.rs @@ -116,7 +116,7 @@ fn eval( .arg(keys) .arg(arguments) .query(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } impl ScriptCache for RedisCache diff --git a/litellm-rust/crates/cache-redis/src/store.rs b/litellm-rust/crates/cache-redis/src/store.rs index 111d481ad02..daadc940ca9 100644 --- a/litellm-rust/crates/cache-redis/src/store.rs +++ b/litellm-rust/crates/cache-redis/src/store.rs @@ -32,7 +32,7 @@ where self.execute(|connection| { connection .set_ex::<_, _, ()>(key, payload, ttl) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) } @@ -41,7 +41,7 @@ where let value = self.execute(|connection| { connection .get::<_, redis::Value>(key) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) })?; self.decode_response(value) } @@ -58,7 +58,7 @@ where self.run(move |connection| { connection .set_ex::<_, _, ()>(key, payload, ttl) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) .await } @@ -73,7 +73,7 @@ where .run(move |connection| { connection .get::<_, redis::Value>(key) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) }) .await?; self.decode_response(value) @@ -158,7 +158,7 @@ pub(crate) fn mget( redis::cmd("MGET") .arg(keys) .query(connection) - .map_err(|_| Error::Unavailable) + .map_err(Error::unavailable) } impl DeleteCache for RedisCache @@ -168,12 +168,12 @@ where { fn delete_cache(&self, key: &str) -> Result<(), Error> { let key = self.namespaced_key(key); - self.execute(|connection| connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)) + self.execute(|connection| connection.del::<_, ()>(key).map_err(Error::unavailable)) } async fn async_delete_cache(&self, key: &str) -> Result<(), Error> { let key = self.namespaced_key(key); - self.run(move |connection| connection.del::<_, ()>(key).map_err(|_| Error::Unavailable)) + self.run(move |connection| connection.del::<_, ()>(key).map_err(Error::unavailable)) .await } } @@ -188,7 +188,7 @@ where return Ok(0); } let keys = self.namespaced_keys(&keys); - self.run(move |connection| connection.del(keys).map_err(|_| Error::Unavailable)) + self.run(move |connection| connection.del(keys).map_err(Error::unavailable)) .await } } @@ -225,7 +225,7 @@ fn flush_matching(connection: &mut ConnectionRef<'_>, pattern: &str) -> Result<( if !keys.is_empty() { connection .del::<_, usize>(keys) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; } Ok(true) }) diff --git a/litellm-rust/crates/cache-redis/tests/cache.rs b/litellm-rust/crates/cache-redis/tests/cache.rs index 70baeffd572..9ec85251733 100644 --- a/litellm-rust/crates/cache-redis/tests/cache.rs +++ b/litellm-rust/crates/cache-redis/tests/cache.rs @@ -178,11 +178,11 @@ async fn codec_errors_propagate_without_writing_partial_batches(context: ExactCa ]); assert_eq!( cache.set_cache("invalid", 255, &context), - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); assert_eq!( cache.async_set_cache("invalid", 255, context.clone()).await, - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); assert_eq!( cache @@ -191,7 +191,7 @@ async fn codec_errors_propagate_without_writing_partial_batches(context: ExactCa context.clone(), ) .await, - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); assert_eq!( cache @@ -200,15 +200,15 @@ async fn codec_errors_propagate_without_writing_partial_batches(context: ExactCa ("invalid".into(), 255, None), ]) .await, - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); assert_eq!( cache.get_cache("invalid", &context), - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); assert_eq!( cache.async_get_cache("invalid", &context).await, - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); } @@ -516,7 +516,7 @@ async fn sets_add_members_and_arm_the_default_ttl() { cache .async_set_cache_sadd("members", Vec::new(), None) .await, - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); } @@ -547,7 +547,7 @@ async fn queues_push_and_pop_namespaced_lists() { ); assert_eq!( cache.async_rpush("queue", Vec::new()).await, - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); assert_eq!( cache.async_lpop("queue", Some(2)).await.unwrap(), @@ -673,15 +673,15 @@ async fn rpush_and_trim_raises_when_a_queued_command_fails() { )]) .with_namespace(Some("ns".into())); - assert_eq!( + assert!(matches!( cache .async_rpush_and_trim("buf", vec!["c".into(), "d".into()], 3) .await, - Err(Error::Unavailable) - ); + Err(Error::Unavailable(Some(_))) + )); assert_eq!( cache.async_rpush_and_trim("buf", Vec::new(), 3).await, - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); } @@ -1122,24 +1122,30 @@ async fn test_connection_reports_success_with_the_python_message() { } #[rstest] -#[case::unexpected_reply(Ok("NOPE"), "Redis ping returned False", false)] +#[case::unexpected_reply(Ok("NOPE"), "Redis ping returned False", None)] #[case::connection_refused( Err(redis::RedisError::from((redis::ErrorKind::Io, "connection refused"))), "Redis connection failed:", - true + Some("connection refused") )] #[tokio::test] async fn test_connection_failures_use_the_python_result_contract( #[case] reply: redis::RedisResult<&'static str>, #[case] message: &str, - #[case] has_error: bool, + #[case] cause: Option<&str>, ) { let cache = json_cache(vec![MockCmd::new(redis::cmd("PING"), reply)]); let result = cache.test_connection().await.unwrap(); assert_eq!(result.status, CacheConnectionStatus::Failed); assert!(result.message.starts_with(message), "{}", result.message); - assert_eq!(result.error.is_some(), has_error); + assert_eq!(result.error.is_some(), cause.is_some()); + assert!(cause.is_none_or(|cause| { + result + .error + .as_deref() + .is_some_and(|error| error.contains(cause)) + })); } #[rstest] diff --git a/litellm-rust/crates/cache-redis/tests/cluster.rs b/litellm-rust/crates/cache-redis/tests/cluster.rs index c1a9a70a5d2..e1ba01bcb04 100644 --- a/litellm-rust/crates/cache-redis/tests/cluster.rs +++ b/litellm-rust/crates/cache-redis/tests/cluster.rs @@ -66,7 +66,7 @@ fn constructor_rejects_unusable_cluster_configs( JsonCodec::new(), ) .err(); - assert!(matches!(error, Some(Error::Unavailable))); + assert!(matches!(error, Some(Error::Unavailable(_)))); } #[rstest] diff --git a/litellm-rust/crates/cache-redis/tests/support/mod.rs b/litellm-rust/crates/cache-redis/tests/support/mod.rs index bff1e618582..b04e1c29f3b 100644 --- a/litellm-rust/crates/cache-redis/tests/support/mod.rs +++ b/litellm-rust/crates/cache-redis/tests/support/mod.rs @@ -16,7 +16,7 @@ impl CacheCodec for TaggedByteCodec { fn encode(&self, value: &u8) -> Result, Error> { if *value > 127 { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } Ok(vec![self.0, *value]) } @@ -24,7 +24,7 @@ impl CacheCodec for TaggedByteCodec { fn decode(&self, bytes: &[u8]) -> Result { match bytes { [tag, value] if *tag == self.0 => Ok(*value), - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } } diff --git a/litellm-rust/crates/cache-response/src/buffer.rs b/litellm-rust/crates/cache-response/src/buffer.rs index 68f2348e28b..deb1a56b503 100644 --- a/litellm-rust/crates/cache-response/src/buffer.rs +++ b/litellm-rust/crates/cache-response/src/buffer.rs @@ -26,7 +26,7 @@ impl WriteBuffer { now: Duration, ) -> Result<(), Error> { let pending = { - let mut entries = self.entries.lock().map_err(|_| Error::Unavailable)?; + let mut entries = self.entries.lock()?; entries.push((request.clone(), response, now)); (entries.len() >= self.flush_size).then(|| std::mem::take(&mut *entries)) }; @@ -39,7 +39,7 @@ impl WriteBuffer { } pub fn clear(&self) -> Result<(), Error> { - self.entries.lock().map_err(|_| Error::Unavailable)?.clear(); + self.entries.lock()?.clear(); Ok(()) } } diff --git a/litellm-rust/crates/cache-response/src/codec.rs b/litellm-rust/crates/cache-response/src/codec.rs index 6b0f29e0a58..1adebe65bc3 100644 --- a/litellm-rust/crates/cache-response/src/codec.rs +++ b/litellm-rust/crates/cache-response/src/codec.rs @@ -14,24 +14,24 @@ impl CacheCodec for ResponseCacheCodec { .timestamp .is_some_and(|timestamp| !timestamp.is_finite()) { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } // Python reads a `response` that is either a dict or a serialized string, so every // other shape is written serialized. A string on the wire is therefore always a // serialized response, which keeps string-valued responses unambiguous. if value.timestamp.is_none() || value.response.is_object() { - return serde_json::to_vec(value).map_err(|_| Error::InvalidEntry); + return serde_json::to_vec(value).map_err(Error::invalid_entry); } - let response = serde_json::to_string(&value.response).map_err(|_| Error::InvalidEntry)?; + let response = serde_json::to_string(&value.response).map_err(Error::invalid_entry)?; serde_json::to_vec(&CacheEntry { timestamp: value.timestamp, response: Value::String(response), }) - .map_err(|_| Error::InvalidEntry) + .map_err(Error::invalid_entry) } fn decode(&self, bytes: &[u8]) -> Result { - let text = std::str::from_utf8(bytes).map_err(|_| Error::InvalidEntry)?; + let text = std::str::from_utf8(bytes).map_err(Error::invalid_entry)?; let value = decode_value(text)?; let Some(timestamp) = value.get("timestamp") else { return Ok(CacheEntry { @@ -40,9 +40,9 @@ impl CacheCodec for ResponseCacheCodec { }); }; let Some(timestamp) = timestamp.as_f64().filter(|timestamp| timestamp.is_finite()) else { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); }; - let response = match value.get("response").ok_or(Error::InvalidEntry)? { + let response = match value.get("response").ok_or(Error::InvalidEntry(None))? { Value::String(text) => decode_value(text)?, response => response.clone(), }; @@ -58,25 +58,25 @@ fn decode_value(text: &str) -> Result { return Ok(value); } check_literal_depth(text)?; - let literal: py_literal::Value = text.parse().map_err(|_| Error::InvalidEntry)?; + let literal: py_literal::Value = text.parse().map_err(Error::invalid_entry)?; literal_value(literal, 0) } fn literal_value(value: py_literal::Value, depth: usize) -> Result { use py_literal::Value as Literal; if depth > 128 { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } match value { Literal::String(text) => Ok(Value::String(text)), Literal::Boolean(value) => Ok(Value::Bool(value)), Literal::None => Ok(Value::Null), Literal::Integer(value) => { - serde_json::from_str(&value.to_string()).map_err(|_| Error::InvalidEntry) + serde_json::from_str(&value.to_string()).map_err(Error::invalid_entry) } Literal::Float(value) => serde_json::Number::from_f64(value) .map(Value::Number) - .ok_or(Error::InvalidEntry), + .ok_or(Error::InvalidEntry(None)), Literal::List(values) | Literal::Tuple(values) => values .into_iter() .map(|value| literal_value(value, depth + 1)) @@ -86,13 +86,13 @@ fn literal_value(value: py_literal::Value, depth: usize) -> Result .into_iter() .map(|(key, value)| { let Literal::String(key) = key else { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); }; Ok((key, literal_value(value, depth + 1)?)) }) .collect::, _>>() .map(Value::Object), - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } @@ -118,7 +118,7 @@ fn check_literal_depth(text: &str) -> Result<(), Error> { '[' | '{' | '(' => { depth += 1; if depth > 128 { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } } ']' | '}' | ')' => depth = depth.saturating_sub(1), diff --git a/litellm-rust/crates/cache-response/src/response.rs b/litellm-rust/crates/cache-response/src/response.rs index a5bbef99a3a..1505297315f 100644 --- a/litellm-rust/crates/cache-response/src/response.rs +++ b/litellm-rust/crates/cache-response/src/response.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use litellm_cache::{ BaseCache, BatchCache, BatchEntry, CacheConnectionResult, CacheContext, ConnectionCache, Error, - FlushCache, + FlushCache, ShortBatch, semantic::{SemanticCache, SemanticLookup}, }; use serde_json::Value; @@ -114,7 +114,7 @@ where .get_cache(&cache_key(&request.key), &request.context) { Ok(entry) => entry, - Err(Error::InvalidEntry) => None, + Err(Error::InvalidEntry(_)) => None, Err(error) => return Err(error), }; Ok(Self::fresh_or_miss(entry, now, request.max_age)) @@ -134,7 +134,7 @@ where .await { Ok(entry) => entry, - Err(Error::InvalidEntry) => None, + Err(Error::InvalidEntry(_)) => None, Err(error) => return Err(error), }; Ok(Self::fresh_or_miss(entry, now, request.max_age)) @@ -338,7 +338,10 @@ where now: Duration, ) -> Result { if readable.len() != entries.len() { - return Err(Error::Unavailable); + return Err(Error::unavailable(ShortBatch { + requested: readable.len(), + returned: entries.len(), + })); } let mut values = vec![None; requests.len()]; for ((index, request), entry) in readable.into_iter().zip(entries) { @@ -361,7 +364,7 @@ where value: Self::fresh_or_miss(lookup.value, now, max_age), similarity: lookup.similarity, }), - Err(Error::InvalidEntry) => Ok(SemanticLookup::miss(None)), + Err(Error::InvalidEntry(_)) => Ok(SemanticLookup::miss(None)), Err(error) => Err(error), } } diff --git a/litellm-rust/crates/cache-response/tests/codec.rs b/litellm-rust/crates/cache-response/tests/codec.rs index 4fb5a5094cb..289465fcda9 100644 --- a/litellm-rust/crates/cache-response/tests/codec.rs +++ b/litellm-rust/crates/cache-response/tests/codec.rs @@ -44,25 +44,23 @@ fn decode_reads_every_python_envelope(#[case] bytes: &[u8], #[case] response: Va #[case::non_utf8(vec![0xff, 0xfe])] #[case::deep_nesting(format!("{}None{}", "[".repeat(1000), "]".repeat(1000)).into_bytes())] fn decode_rejects_invalid_entries(#[case] bytes: Vec) { - assert_eq!( - ResponseCacheCodec.decode(&bytes).unwrap_err(), - Error::InvalidEntry - ); + assert!(matches!( + ResponseCacheCodec.decode(&bytes), + Err(Error::InvalidEntry(_)) + )); } #[rstest] #[case::nan(f64::NAN)] #[case::infinity(f64::INFINITY)] fn encode_rejects_non_finite_timestamps(#[case] timestamp: f64) { - assert_eq!( - ResponseCacheCodec - .encode(&CacheEntry { - timestamp: Some(timestamp), - response: json!({}), - }) - .unwrap_err(), - Error::InvalidEntry - ); + assert!(matches!( + ResponseCacheCodec.encode(&CacheEntry { + timestamp: Some(timestamp), + response: json!({}), + }), + Err(Error::InvalidEntry(_)) + )); } #[rstest] diff --git a/litellm-rust/crates/cache-response/tests/response.rs b/litellm-rust/crates/cache-response/tests/response.rs index 86ddf4a72cd..70ae259d33a 100644 --- a/litellm-rust/crates/cache-response/tests/response.rs +++ b/litellm-rust/crates/cache-response/tests/response.rs @@ -158,8 +158,16 @@ fn scored(timestamp: f64, similarity: f64) -> Result, )] #[case::no_search(Ok(SemanticLookup::miss(None)), true, Ok(SemanticLookup::miss(None)))] #[case::disabled_reads_skip_the_backend(scored(95.0, 0.95), false, Ok(SemanticLookup::miss(None)))] -#[case::invalid_entry_is_a_miss(Err(Error::InvalidEntry), true, Ok(SemanticLookup::miss(None)))] -#[case::backend_errors_propagate(Err(Error::Unavailable), true, Err(Error::Unavailable))] +#[case::invalid_entry_is_a_miss( + Err(Error::InvalidEntry(None)), + true, + Ok(SemanticLookup::miss(None)) +)] +#[case::backend_errors_propagate( + Err(Error::Unavailable(None)), + true, + Err(Error::Unavailable(None)) +)] #[tokio::test] async fn semantic_lookup_applies_freshness_to_the_value_only( #[case] backend: Result, Error>, diff --git a/litellm-rust/crates/cache-s3/Cargo.toml b/litellm-rust/crates/cache-s3/Cargo.toml index eb3a2fff1ac..d51c282d4d7 100644 --- a/litellm-rust/crates/cache-s3/Cargo.toml +++ b/litellm-rust/crates/cache-s3/Cargo.toml @@ -20,6 +20,7 @@ reqwest.workspace = true tokio.workspace = true [dev-dependencies] +litellm-error.workspace = true litellm-http = { workspace = true, features = ["test-support"] } litellm-cache-testing.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/cache-s3/src/auth.rs b/litellm-rust/crates/cache-s3/src/auth.rs index f4f06e5371d..659390a5659 100644 --- a/litellm-rust/crates/cache-s3/src/auth.rs +++ b/litellm-rust/crates/cache-s3/src/auth.rs @@ -46,7 +46,7 @@ impl ProvideCredentials for S3Credentials { self.auth .resolve_credentials(self.config.clone(), &self.env) .await - .map_err(|_| CredentialsError::provider_error("S3 cache authentication failed")) + .map_err(CredentialsError::provider_error) }) } } diff --git a/litellm-rust/crates/cache-s3/src/cache.rs b/litellm-rust/crates/cache-s3/src/cache.rs index ced948e80c6..5369bca0f18 100644 --- a/litellm-rust/crates/cache-s3/src/cache.rs +++ b/litellm-rust/crates/cache-s3/src/cache.rs @@ -125,7 +125,7 @@ impl S3Cache { } None => request.cache_control("immutable, max-age=31536000, s-maxage=31536000"), }; - request.send().await.map_err(|_| Error::Unavailable)?; + request.send().await.map_err(Error::unavailable)?; Ok(()) } @@ -152,7 +152,7 @@ impl S3Cache { return Ok(None); } } - return Err(Error::Unavailable); + return Err(Error::unavailable(error)); } }; if let Some(expires) = output.expires_string() @@ -165,7 +165,7 @@ impl S3Cache { .body .collect() .await - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? .into_bytes(); self.codec.decode(&bytes).map(Some) } diff --git a/litellm-rust/crates/cache-s3/src/transport.rs b/litellm-rust/crates/cache-s3/src/transport.rs index eabc54ac9e2..5b6e867ede5 100644 --- a/litellm-rust/crates/cache-s3/src/transport.rs +++ b/litellm-rust/crates/cache-s3/src/transport.rs @@ -32,7 +32,9 @@ impl HttpConnector for ReqwestHttpClient { let request = reqwest::Request::try_from(request) .map_err(|error| ConnectorError::other(error.into(), None))?; let response = client.execute(request).await.map_err(|error| { - if error.is_timeout() { + let timeout = error.is_timeout(); + let error = litellm_http::Error::from(error); + if timeout { ConnectorError::timeout(error.into()) } else { ConnectorError::io(error.into()) diff --git a/litellm-rust/crates/cache-s3/tests/cache.rs b/litellm-rust/crates/cache-s3/tests/cache.rs index 53a51f8a289..d0469620493 100644 --- a/litellm-rust/crates/cache-s3/tests/cache.rs +++ b/litellm-rust/crates/cache-s3/tests/cache.rs @@ -147,15 +147,15 @@ async fn async_set_signs_with_the_configured_keys(#[future(awt)] server: MockSer #[case::malformed( "malformed", ResponseTemplate::new(200).set_body_string("not a cache entry"), - Err(Error::InvalidEntry) + Err("invalid cache entry: ") )] -#[case::server_error("broken", ResponseTemplate::new(500), Err(Error::Unavailable))] +#[case::server_error("broken", ResponseTemplate::new(500), Err("cache is unavailable: "))] #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn get_maps_s3_responses( #[future(awt)] server: MockServer, #[case] key: &str, #[case] response: ResponseTemplate, - #[case] expected: Result, Error>, + #[case] expected: Result, &str>, ) { Mock::given(method("GET")) .and(path(format!("/cache-bucket/team/{key}"))) @@ -164,9 +164,21 @@ async fn get_maps_s3_responses( .await; let cache = support::cache(&server.uri()); let context = ExactCacheContext::default(); - - assert_eq!(cache.get_cache(key, &context), expected); - assert_eq!(cache.async_get_cache(key, &context).await, expected); + let outcome = |result: Result, Error>| { + result.map_err(|error| litellm_error::Report::new(&error).to_string()) + }; + let sync = outcome(cache.get_cache(key, &context)); + let asynchronous = outcome(cache.async_get_cache(key, &context).await); + match expected { + Ok(value) => { + assert_eq!(sync, Ok(value.clone())); + assert_eq!(asynchronous, Ok(value)); + } + Err(prefix) => { + assert!(sync.unwrap_err().starts_with(prefix)); + assert!(asynchronous.unwrap_err().starts_with(prefix)); + } + } } #[rstest] diff --git a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml index da9bb60c853..a2803b8271f 100644 --- a/litellm-rust/crates/cache-valkey-semantic/Cargo.toml +++ b/litellm-rust/crates/cache-valkey-semantic/Cargo.toml @@ -10,9 +10,11 @@ litellm-cache.workspace = true litellm-cache-redis.workspace = true redis = { version = "1.7.0", features = ["tls-rustls"] } sha2.workspace = true +thiserror.workspace = true uuid = { version = "1", features = ["v4"] } [dev-dependencies] +litellm-error.workspace = true litellm-cache-testing.workspace = true redis-test = "1.0.4" rstest.workspace = true diff --git a/litellm-rust/crates/cache-valkey-semantic/src/index.rs b/litellm-rust/crates/cache-valkey-semantic/src/index.rs index a0b7be4ca7b..3dfff32ff7c 100644 --- a/litellm-rust/crates/cache-valkey-semantic/src/index.rs +++ b/litellm-rust/crates/cache-valkey-semantic/src/index.rs @@ -23,8 +23,7 @@ pub(crate) fn ensure_index( ) -> Result<(), Error> { if index .dimension - .lock() - .map_err(|_| Error::Unavailable)? + .lock()? .is_some_and(|existing| existing == dimension) { return Ok(()); @@ -50,24 +49,42 @@ pub(crate) fn ensure_index( .arg("DISTANCE_METRIC") .arg("COSINE") .query::(connection) - .map(|_| ()) - .map_err(|error| error.to_string()); - if let Err(message) = create { - if !message.to_ascii_lowercase().contains("already exists") { - return Err(Error::Unavailable); + .map(|_| ()); + if let Err(error) = create { + if !index_exists(&error) { + return Err(Error::unavailable(error)); } let info = redis::cmd("FT.INFO") .arg(&index.name) .query::(connection) - .map_err(|_| Error::Unavailable)?; - if index_dimension_from_info(&info).is_some_and(|existing| existing != dimension) { - return Err(Error::Unavailable); + .map_err(Error::unavailable)?; + if let Some(existing) = + index_dimension_from_info(&info).filter(|existing| *existing != dimension) + { + return Err(Error::unavailable(DimensionMismatch { + existing, + dimension, + })); } } - *index.dimension.lock().map_err(|_| Error::Unavailable)? = Some(dimension); + *index.dimension.lock()? = Some(dimension); Ok(()) } +fn index_exists(error: &redis::RedisError) -> bool { + error.code().is_some() + && error + .detail() + .is_some_and(|detail| detail.to_ascii_lowercase().contains("already exists")) +} + +#[derive(Debug, thiserror::Error)] +#[error("index exists with dimension {existing}, not {dimension}")] +struct DimensionMismatch { + existing: usize, + dimension: usize, +} + /// `_extract_index_dim`: flatten each attribute one level and read the value after /// `dimensions`. fn index_dimension_from_info(value: &redis::Value) -> Option { diff --git a/litellm-rust/crates/cache-valkey-semantic/src/search.rs b/litellm-rust/crates/cache-valkey-semantic/src/search.rs index 3b4fae84317..c5e4e748b58 100644 --- a/litellm-rust/crates/cache-valkey-semantic/src/search.rs +++ b/litellm-rust/crates/cache-valkey-semantic/src/search.rs @@ -53,9 +53,7 @@ pub(crate) fn write_document( .arg(ttl.as_secs()) .ignore(); } - pipeline - .query::<()>(connection) - .map_err(|_| Error::Unavailable) + pipeline.query::<()>(connection).map_err(Error::unavailable) } /// The KNN-1 search within `scope`: the closest document's similarity, and its stored response @@ -83,7 +81,7 @@ pub(crate) fn search_document( .arg("DIALECT") .arg(2) .query::(connection) - .map_err(|_| Error::Unavailable)?; + .map_err(Error::unavailable)?; let Some(fields) = search_fields(response)? else { return Ok(SemanticLookup::miss(Some(0.0))); }; @@ -91,7 +89,7 @@ pub(crate) fn search_document( fields .iter() .find_map(|(field, value)| (field == name).then(|| value.clone())) - .ok_or(Error::InvalidEntry) + .ok_or(Error::InvalidEntry(None)) }; let response = field("response")?; let similarity = 1.0 - parse_f64(&field("vector_distance")?)?; @@ -105,24 +103,24 @@ type SearchFields = Vec<(String, Vec)>; fn search_fields(value: redis::Value) -> Result, Error> { let redis::Value::Array(values) = value else { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); }; - let total = parse_i64(values.first().ok_or(Error::InvalidEntry)?)?; + let total = parse_i64(values.first().ok_or(Error::InvalidEntry(None))?)?; if total <= 0 || values.len() < 3 { return Ok(None); } let redis::Value::Array(fields) = &values[2] else { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); }; let (pairs, remainder) = fields.as_chunks::<2>(); if !remainder.is_empty() { - return Err(Error::InvalidEntry); + return Err(Error::InvalidEntry(None)); } pairs .iter() .map(|pair| { Ok(( - value_text(&pair[0]).ok_or(Error::InvalidEntry)?, + value_text(&pair[0]).ok_or(Error::InvalidEntry(None))?, value_bytes(&pair[1])?, )) }) @@ -132,16 +130,16 @@ fn search_fields(value: redis::Value) -> Result, Error> { fn parse_i64(value: &redis::Value) -> Result { value_text(value) - .ok_or(Error::InvalidEntry)? + .ok_or(Error::InvalidEntry(None))? .parse() - .map_err(|_| Error::InvalidEntry) + .map_err(Error::invalid_entry) } fn parse_f64(value: &[u8]) -> Result { std::str::from_utf8(value) - .map_err(|_| Error::InvalidEntry)? + .map_err(Error::invalid_entry)? .parse() - .map_err(|_| Error::InvalidEntry) + .map_err(Error::invalid_entry) } pub(crate) fn value_text(value: &redis::Value) -> Option { @@ -158,6 +156,6 @@ fn value_bytes(value: &redis::Value) -> Result, Error> { redis::Value::BulkString(bytes) => Ok(bytes.clone()), redis::Value::SimpleString(value) => Ok(value.as_bytes().to_vec()), redis::Value::Int(value) => Ok(value.to_string().into_bytes()), - _ => Err(Error::InvalidEntry), + _ => Err(Error::InvalidEntry(None)), } } diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs index 1c693d0875d..10e32c0183b 100644 --- a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs @@ -83,10 +83,7 @@ fn ok() -> redis::RedisResult { } fn already_exists() -> redis::RedisResult { - Err(redis::RedisError::from(( - redis::ErrorKind::Io, - "already exists", - ))) + Err(support::server_error("Index already exists")) } fn bulk(value: &[u8]) -> redis::Value { @@ -250,19 +247,29 @@ fn second_set_skips_create_after_dimension_is_cached(entry: Value, context: Sema #[rstest] #[case::nested_matching(nested_dimension_info(3), Ok(()))] -#[case::nested_mismatch(nested_dimension_info(2), Err(Error::Unavailable))] +#[case::nested_mismatch( + nested_dimension_info(2), + Err("cache is unavailable: index exists with dimension 2, not 3") +)] #[case::flat_matching(flat_dimension_info("3"), Ok(()))] -#[case::flat_mismatch(flat_dimension_info("2"), Err(Error::Unavailable))] +#[case::flat_mismatch( + flat_dimension_info("2"), + Err("cache is unavailable: index exists with dimension 2, not 3") +)] #[case::unreported_dimension_is_accepted(redis::Value::Array(vec![]), Ok(()))] fn existing_index_dimension_must_match_embedding( #[case] info: redis::Value, - #[case] expected: Result<(), Error>, + #[case] expected: Result<(), &str>, entry: Value, context: SemanticCacheContext, ) { let recording = recording([already_exists(), Ok(info)]); - assert_eq!(recording.cache.set_cache("key", entry, &context), expected); + let result = recording.cache.set_cache("key", entry, &context); + assert_eq!( + result.map_err(|error| litellm_error::Report::new(&error).to_string()), + expected.map_err(str::to_owned) + ); } #[rstest] @@ -274,10 +281,10 @@ fn index_creation_failures_are_unavailable( ) { let recording = recording([reply]); - assert_eq!( + assert!(matches!( recording.cache.set_cache("key", entry, &context), - Err(Error::Unavailable) - ); + Err(Error::Unavailable(Some(_))) + )); } #[rstest] @@ -291,10 +298,10 @@ fn index_creation_failures_are_unavailable( bulk(b"document"), redis::Value::Array(vec![bulk(b"vector_distance"), bulk(b"0.1")]), ]), - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) )] -#[case::unparsable_distance(search_hit(b"not-json", "abc"), Err(Error::InvalidEntry))] -#[case::undecodable_response(search_hit(b"not-json", "0.1"), Err(Error::InvalidEntry))] +#[case::unparsable_distance(search_hit(b"not-json", "abc"), Err(Error::InvalidEntry(None)))] +#[case::undecodable_response(search_hit(b"not-json", "0.1"), Err(Error::InvalidEntry(None)))] fn get_applies_threshold_and_decodes_entry( #[case] reply: redis::Value, #[case] expected: Result, Error>, @@ -302,7 +309,13 @@ fn get_applies_threshold_and_decodes_entry( ) { let recording = recording([ok(), Ok(reply)]); - assert_eq!(recording.cache.get_cache("key", &context), expected); + let kind = |result: Result, Error>| { + result.map_err(|error| std::mem::discriminant(&error)) + }; + assert_eq!( + kind(recording.cache.get_cache("key", &context)), + kind(expected) + ); } #[rstest] diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs b/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs index 2c6ff06b134..2af05943c98 100644 --- a/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs +++ b/litellm-rust/crates/cache-valkey-semantic/tests/support/mod.rs @@ -74,7 +74,7 @@ impl FakeSearch { "FT.CREATE" => { let name = text(1); if state.indexes.contains_key(&name) { - return Err(error("Index already exists")); + return Err(server_error("Index already exists")); } let position = |token: &str| args.iter().position(|arg| arg == token.as_bytes()); let prefix = args[position("PREFIX").unwrap() + 2].clone(); @@ -94,7 +94,7 @@ impl FakeSearch { let index = state .indexes .get(&text(1)) - .ok_or_else(|| error("Unknown index name"))?; + .ok_or_else(|| server_error("Unknown index name"))?; Ok(index_info(index)) } "FT.DROPINDEX" => { @@ -118,7 +118,7 @@ impl FakeSearch { let index = state .indexes .get(&text(1)) - .ok_or_else(|| error("no such index"))?; + .ok_or_else(|| server_error("no such index"))?; let query = text(2); let tag = query_tag(&query); let params = args.iter().position(|arg| arg == b"PARAMS").unwrap(); @@ -155,7 +155,7 @@ impl FakeSearch { ])) } "PING" => Ok(redis::Value::SimpleString("PONG".into())), - _ => Err(error("unsupported command")), + _ => Err(server_error("unsupported command")), } } } @@ -192,8 +192,11 @@ impl redis::ConnectionLike for FakeSearch { } } -fn error(message: &'static str) -> redis::RedisError { - redis::RedisError::from((redis::ErrorKind::Extension, message)) +pub fn server_error(reply: &str) -> redis::RedisError { + redis::parse_redis_value(format!("-{reply}\r\n").as_bytes()) + .expect("error reply") + .extract_error() + .expect_err("server error") } fn bulk(bytes: &[u8]) -> redis::Value { diff --git a/litellm-rust/crates/cache/Cargo.toml b/litellm-rust/crates/cache/Cargo.toml index 51bbe12a85e..282c34e6b02 100644 --- a/litellm-rust/crates/cache/Cargo.toml +++ b/litellm-rust/crates/cache/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-error.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } strum.workspace = true diff --git a/litellm-rust/crates/cache/src/capabilities.rs b/litellm-rust/crates/cache/src/capabilities.rs index ac9d8fc8764..b30f9e162f2 100644 --- a/litellm-rust/crates/cache/src/capabilities.rs +++ b/litellm-rust/crates/cache/src/capabilities.rs @@ -48,7 +48,7 @@ pub trait BatchCache: BaseCache { .map(|key| match self.get_cache(key, context) { Ok(Some(value)) => Ok(BatchEntry::Hit(value)), Ok(None) => Ok(BatchEntry::Miss), - Err(Error::InvalidEntry) => Ok(BatchEntry::Invalid), + Err(Error::InvalidEntry(_)) => Ok(BatchEntry::Invalid), Err(error) => Err(error), }) .collect() @@ -65,7 +65,7 @@ pub trait BatchCache: BaseCache { entries.push(match self.async_get_cache(&key, &context).await { Ok(Some(value)) => BatchEntry::Hit(value), Ok(None) => BatchEntry::Miss, - Err(Error::InvalidEntry) => BatchEntry::Invalid, + Err(Error::InvalidEntry(_)) => BatchEntry::Invalid, Err(error) => return Err(error), }); } diff --git a/litellm-rust/crates/cache/src/codec.rs b/litellm-rust/crates/cache/src/codec.rs index 6d47c682406..b802e40a8b3 100644 --- a/litellm-rust/crates/cache/src/codec.rs +++ b/litellm-rust/crates/cache/src/codec.rs @@ -41,10 +41,10 @@ where type Value = V; fn encode(&self, value: &Self::Value) -> Result, Error> { - serde_json::to_vec(value).map_err(|_| Error::InvalidEntry) + serde_json::to_vec(value).map_err(Error::invalid_entry) } fn decode(&self, bytes: &[u8]) -> Result { - serde_json::from_slice(bytes).map_err(|_| Error::InvalidEntry) + serde_json::from_slice(bytes).map_err(Error::invalid_entry) } } diff --git a/litellm-rust/crates/cache/src/dual.rs b/litellm-rust/crates/cache/src/dual.rs index c4bc953239d..3deee2560f4 100644 --- a/litellm-rust/crates/cache/src/dual.rs +++ b/litellm-rust/crates/cache/src/dual.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use crate::{ BaseCache, BatchCache, BatchEntry, BulkDeleteCache, CacheContext, ClaimCache, CounterCache, - DeleteCache, Error, FlushCache, IncrementOperation, SetCache, TtlCache, + DeleteCache, Error, FlushCache, IncrementOperation, SetCache, ShortBatch, TtlCache, }; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -98,7 +98,7 @@ impl DualCache { fn remote(&self, result: Result) -> Result, Error> { match result { Ok(value) => Ok(Some(value)), - Err(Error::Unavailable) + Err(Error::Unavailable(_)) if self.remote_failure_policy == RemoteFailurePolicy::UseLocal => { Ok(None) @@ -136,7 +136,10 @@ where remote: Vec>, ) -> Result>, Error> { if missing.len() != remote.len() { - return Err(Error::Unavailable); + return Err(Error::unavailable(ShortBatch { + requested: missing.len(), + returned: remote.len(), + })); } for (index, entry) in missing.into_iter().zip(remote) { if let BatchEntry::Hit(value) = &entry { @@ -389,7 +392,10 @@ where } let values = self.l2.async_increment_pipeline(operations.clone()).await?; if values.len() != operations.len() { - return Err(Error::Unavailable); + return Err(Error::unavailable(ShortBatch { + requested: operations.len(), + returned: values.len(), + })); } for (operation, value) in operations.iter().zip(&values) { self.l1 diff --git a/litellm-rust/crates/cache/src/error.rs b/litellm-rust/crates/cache/src/error.rs index 79ef9cd18b1..8861651a1c8 100644 --- a/litellm-rust/crates/cache/src/error.rs +++ b/litellm-rust/crates/cache/src/error.rs @@ -1,9 +1,11 @@ +use litellm_error::Source; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("cache is unavailable")] - Unavailable, + Unavailable(#[source] Option), #[error("invalid cache entry")] - InvalidEntry, + InvalidEntry(#[source] Option), #[error("flushing Redis requires an explicit namespace")] UnscopedFlush, #[error("operation is not supported by this cache")] @@ -11,3 +13,59 @@ pub enum Error { #[error("semantic cache requires request messages")] MissingPrompt, } + +#[derive(Debug, thiserror::Error)] +#[error("cache backend returned {returned} results for {requested} keys")] +pub struct ShortBatch { + pub requested: usize, + pub returned: usize, +} + +#[derive(Debug, thiserror::Error)] +#[error("cache lock is poisoned")] +struct Poisoned; + +impl From> for Error { + fn from(_: std::sync::PoisonError) -> Self { + Self::unavailable(Poisoned) + } +} + +impl Error { + pub fn unavailable(cause: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Unavailable(Some(Source::new(cause))) + } + + pub fn invalid_entry(cause: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::InvalidEntry(Some(Source::new(cause))) + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::Error; + + #[rstest] + #[case::unavailable( + Error::unavailable(std::io::Error::other("connection refused")), + "cache is unavailable" + )] + #[case::invalid_entry( + Error::invalid_entry(std::io::Error::other("connection refused")), + "invalid cache entry" + )] + fn the_backend_cause_is_the_source_and_rendered_once( + #[case] error: Error, + #[case] message: &str, + ) { + let report = litellm_error::Report::new(&error).to_string(); + assert_eq!(report, format!("{message}: connection refused")); + assert!(!error.to_string().contains("connection refused")); + let cause = std::error::Error::source(&error) + .and_then(|cause| cause.downcast_ref::()) + .expect("cause"); + assert!(cause.downcast_ref::().is_some()); + } +} diff --git a/litellm-rust/crates/cache/src/lib.rs b/litellm-rust/crates/cache/src/lib.rs index 55d8a5fa28b..4e07f3b4e8c 100644 --- a/litellm-rust/crates/cache/src/lib.rs +++ b/litellm-rust/crates/cache/src/lib.rs @@ -23,4 +23,4 @@ pub use codec::{CacheCodec, JsonCodec}; pub use dual::{ DEFAULT_DELETE_BATCH_SIZE, DualCache, ReadPolicy, RemoteFailurePolicy, WritePolicy, }; -pub use error::Error; +pub use error::{Error, ShortBatch}; diff --git a/litellm-rust/crates/cache/tests/caching.rs b/litellm-rust/crates/cache/tests/caching.rs index baf968f4659..84a3b8c4d2a 100644 --- a/litellm-rust/crates/cache/tests/caching.rs +++ b/litellm-rust/crates/cache/tests/caching.rs @@ -67,7 +67,7 @@ impl BaseCache for TestCache { } fn set_cache(&self, _: &str, _: Self::Value, _: &ExactCacheContext) -> Result<(), Error> { - Err(Error::Unavailable) + Err(Error::Unavailable(None)) } async fn async_set_cache( @@ -77,7 +77,7 @@ impl BaseCache for TestCache { context: ExactCacheContext, ) -> Result<(), Error> { if key == "unavailable" { - return Err(Error::Unavailable); + return Err(Error::Unavailable(None)); } self.writes .lock() @@ -118,7 +118,7 @@ impl BaseCache for RecordingCounter { impl CounterCache for RecordingCounter { fn increment_cache(&self, key: &str, amount: f64, _: ExactCacheContext) -> Result { if key == "unavailable" { - return Err(Error::Unavailable); + return Err(Error::Unavailable(None)); } let mut total = self.total.lock().unwrap(); *total += amount; @@ -226,7 +226,7 @@ async fn default_batch_operations_use_async_writes_and_stop_on_failure(cache: Te context.clone(), ) .await, - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); assert_eq!( *cache.writes.lock().unwrap(), @@ -323,7 +323,7 @@ async fn default_increment_pipeline_stops_at_the_first_failure() { operation("skipped", 1.0, None), ]) .await, - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); let keys = cache .increments diff --git a/litellm-rust/crates/cache/tests/codec.rs b/litellm-rust/crates/cache/tests/codec.rs index 5320545fd49..ff4aeed87e6 100644 --- a/litellm-rust/crates/cache/tests/codec.rs +++ b/litellm-rust/crates/cache/tests/codec.rs @@ -31,12 +31,18 @@ fn json_codec_round_trips_typed_domain_values() { #[case::wrongly_typed(br#"{"deployment":12}"#)] fn json_codec_rejects_malformed_and_wrongly_typed_entries(#[case] bytes: &[u8]) { let codec = JsonCodec::::new(); - assert_eq!(codec.decode(bytes).unwrap_err(), Error::InvalidEntry); + assert!(matches!( + codec.decode(bytes), + Err(Error::InvalidEntry(Some(_))) + )); } #[rstest] fn json_codec_propagates_encoding_errors() { let codec = JsonCodec::>::new(); let value = BTreeMap::from([((1, 2), "invalid JSON object key".into())]); - assert_eq!(codec.encode(&value).unwrap_err(), Error::InvalidEntry); + assert!(matches!( + codec.encode(&value), + Err(Error::InvalidEntry(Some(_))) + )); } diff --git a/litellm-rust/crates/cache/tests/dual.rs b/litellm-rust/crates/cache/tests/dual.rs index 87dd76388ef..86af5e80abb 100644 --- a/litellm-rust/crates/cache/tests/dual.rs +++ b/litellm-rust/crates/cache/tests/dual.rs @@ -71,7 +71,7 @@ where impl CounterCache for TestCache { fn increment_cache(&self, _: &str, amount: f64, _: ExactCacheContext) -> Result { if self.fail { - return Err(Error::Unavailable); + return Err(Error::Unavailable(None)); } let mut value = self.value.lock().unwrap(); let incremented = value.unwrap_or_default() + amount; @@ -92,7 +92,7 @@ where _: ExactCacheContext, ) -> Result { if self.fail { - return Err(Error::Unavailable); + return Err(Error::Unavailable(None)); } let mut value = self.value.lock().unwrap(); let winner = match value.as_ref() { @@ -113,7 +113,7 @@ fn failed_l2_increment_leaves_l1_unchanged() { assert_eq!( cache.increment_cache("counter", 2.0, ExactCacheContext::default()), - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); assert_eq!( l1.get_cache("counter", &ExactCacheContext::default()) @@ -276,11 +276,11 @@ impl BaseCache for Unavailable { } fn set_cache(&self, _: &str, _: String, _: &ExactCacheContext) -> Result<(), Error> { - Err(Error::Unavailable) + Err(Error::Unavailable(None)) } fn get_cache(&self, _: &str, _: &ExactCacheContext) -> Result, Error> { - Err(Error::Unavailable) + Err(Error::Unavailable(None)) } } @@ -288,13 +288,13 @@ impl BatchCache for Unavailable {} impl DeleteCache for Unavailable { fn delete_cache(&self, _: &str) -> Result<(), Error> { - Err(Error::Unavailable) + Err(Error::Unavailable(None)) } } impl FlushCache for Unavailable { fn flush_cache(&self) -> Result<(), Error> { - Err(Error::Unavailable) + Err(Error::Unavailable(None)) } } @@ -306,7 +306,7 @@ impl ClaimCache for Unavailable { _: &[String], _: ExactCacheContext, ) -> Result { - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) } } @@ -316,9 +316,12 @@ fn remote_failure_policy_selects_propagation_or_the_local_tier() { let strict = DualCache::new(Arc::new(TestCache::new(None, false)), Arc::new(Unavailable)); assert_eq!( strict.set_cache("key", "value".into(), &context), - Err(Error::Unavailable) + Err(Error::Unavailable(None)) + ); + assert_eq!( + strict.get_cache("key", &context), + Err(Error::Unavailable(None)) ); - assert_eq!(strict.get_cache("key", &context), Err(Error::Unavailable)); let l1 = Arc::new(TestCache::new(None, false)); let degraded = DualCache::new(l1.clone(), Arc::new(Unavailable)) @@ -347,7 +350,7 @@ fn claim_fallback_does_not_hide_non_availability_errors() { &[], ExactCacheContext::default() ), - Err(Error::InvalidEntry) + Err(Error::InvalidEntry(None)) ); } @@ -419,7 +422,7 @@ impl Tier { fn check(&self) -> Result<(), Error> { if self.fail { - return Err(Error::Unavailable); + return Err(Error::Unavailable(None)); } Ok(()) } @@ -598,7 +601,7 @@ async fn failed_async_l2_increment_leaves_l1_unchanged(log: Log) { cache .async_increment("counter", 2.0, seconds(30), true) .await, - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); assert_eq!( l1.counter("counter"), @@ -669,7 +672,7 @@ async fn failed_l2_increment_pipeline_leaves_l1_unchanged(log: Log) { ttl: None, }]) .await, - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); assert_eq!(events(&log), ["l2 pipeline a"]); assert_eq!(l1.counter("a"), Some((1.0, ExactCacheContext::default()))); @@ -713,7 +716,7 @@ async fn failed_local_set_add_never_reaches_l2(log: Log) { cache .async_set_cache_sadd("members", vec!["a".into()], None) .await, - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); assert!(events(&log).is_empty()); } @@ -768,7 +771,7 @@ async fn bulk_delete_stops_before_l2_when_the_local_delete_fails(log: Log) { ); assert_eq!( cache.delete_cache_keys(vec!["a".into()]).await, - Err(Error::Unavailable) + Err(Error::Unavailable(None)) ); assert!(events(&log).is_empty()); } diff --git a/litellm-rust/crates/config/Cargo.toml b/litellm-rust/crates/config/Cargo.toml index 23d53923362..92c219c0b01 100644 --- a/litellm-rust/crates/config/Cargo.toml +++ b/litellm-rust/crates/config/Cargo.toml @@ -14,6 +14,7 @@ strum.workspace = true thiserror.workspace = true [dev-dependencies] +litellm-error.workspace = true rstest.workspace = true serde_json.workspace = true tempfile.workspace = true diff --git a/litellm-rust/crates/config/src/error.rs b/litellm-rust/crates/config/src/error.rs index 61d19491abc..300f15380bd 100644 --- a/litellm-rust/crates/config/src/error.rs +++ b/litellm-rust/crates/config/src/error.rs @@ -1,7 +1,14 @@ +use std::{io, path::PathBuf}; + #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("could not read config")] - Read(#[from] std::io::Error), + #[error("could not read config {}", path.display())] + Read { path: PathBuf, source: io::Error }, #[error("invalid YAML config")] Parse(#[from] serde_yaml_ng::Error), + #[error("invalid YAML config {}", path.display())] + ParseFile { + path: PathBuf, + source: serde_yaml_ng::Error, + }, } diff --git a/litellm-rust/crates/config/src/includes.rs b/litellm-rust/crates/config/src/includes.rs index 78df8e9a33a..27b908650df 100644 --- a/litellm-rust/crates/config/src/includes.rs +++ b/litellm-rust/crates/config/src/includes.rs @@ -15,11 +15,31 @@ struct Includes { } fn read(path: &Path) -> Result { - Ok(serde_yaml_ng::from_str(&std::fs::read_to_string(path)?)?) + let text = std::fs::read_to_string(path).map_err(|source| Error::Read { + path: path.to_owned(), + source, + })?; + serde_yaml_ng::from_str(&text).map_err(|source| Error::ParseFile { + path: path.to_owned(), + source, + }) +} + +fn canonical(path: &Path) -> Result { + path.canonicalize().map_err(|source| Error::Read { + path: path.to_owned(), + source, + }) } fn entries(config: &Mapping, path: &Path) -> Result, Error> { - let includes: Includes = serde_yaml_ng::from_value(Value::Mapping(config.clone()))?; + let includes: Includes = + serde_yaml_ng::from_value(Value::Mapping(config.clone())).map_err(|source| { + Error::ParseFile { + path: path.to_owned(), + source, + } + })?; Ok(includes .include .into_iter() @@ -28,7 +48,7 @@ fn entries(config: &Mapping, path: &Path) -> Result, Erro } pub(super) fn load(path: &Path) -> Result { - let root = path.canonicalize()?; + let root = canonical(path)?; let mut merged = read(&root)?; let mut pending: VecDeque<_> = entries(&merged, &root)?.into(); let mut loaded = BTreeSet::from([root.clone()]); @@ -36,12 +56,11 @@ pub(super) fn load(path: &Path) -> Result { while let Some((entry, declaring)) = pending.pop_front() { let declared = declaring.parent().unwrap_or(Path::new(".")).join(&entry); let fallback = root.parent().unwrap_or(Path::new(".")).join(&entry); - let location = if declared.exists() { - declared + let location = canonical(if declared.exists() { + &declared } else { - fallback - } - .canonicalize()?; + &fallback + })?; if !loaded.insert(location.clone()) { continue; } diff --git a/litellm-rust/crates/config/tests/config.rs b/litellm-rust/crates/config/tests/config.rs index d425b62efc2..3c7d02203e5 100644 --- a/litellm-rust/crates/config/tests/config.rs +++ b/litellm-rust/crates/config/tests/config.rs @@ -84,9 +84,15 @@ fn rejects_malformed_and_incomplete_config(#[case] yaml: &str) { #[rstest] fn distinguishes_read_errors_from_parse_errors(directory: TempDir) { + let error = Config::load(directory.path().join("missing.yaml")).unwrap_err(); + assert!( + litellm_error::Report::new(&error) + .to_string() + .contains("missing.yaml") + ); assert!(matches!( - Config::load(directory.path().join("missing.yaml")), - Err(Error::Read(error)) if error.kind() == std::io::ErrorKind::NotFound + error, + Error::Read { source, .. } if source.kind() == std::io::ErrorKind::NotFound )); } diff --git a/litellm-rust/crates/core-utils/Cargo.toml b/litellm-rust/crates/core-utils/Cargo.toml index 1feea5fcc08..2e5857c79f3 100644 --- a/litellm-rust/crates/core-utils/Cargo.toml +++ b/litellm-rust/crates/core-utils/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] fancy-regex.workspace = true +litellm-error.workspace = true litellm-tracing.workspace = true litellm-llms-types.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/core-utils/src/call_arguments.rs b/litellm-rust/crates/core-utils/src/call_arguments.rs index 062dad0a92a..3277ca26c7f 100644 --- a/litellm-rust/crates/core-utils/src/call_arguments.rs +++ b/litellm-rust/crates/core-utils/src/call_arguments.rs @@ -1,5 +1,7 @@ use std::ops::Deref; +use litellm_error::Source; + use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::{Map, Value}; @@ -17,9 +19,11 @@ impl CallArguments { } #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] -#[error("invalid argument: {path}")] +#[error("invalid argument {path}")] pub struct ArgumentError { pub path: String, + #[source] + source: Source, } pub fn parse_options(arguments: &CallArguments) -> Result { @@ -28,6 +32,7 @@ pub fn parse_options(arguments: &CallArguments) -> Result( body: &B, consumed: &[&str], ) -> Result { - let Value::Object(fields) = - serde_json::to_value(body).map_err(|_| crate::params::Error::Body)? + let Value::Object(fields) = serde_json::to_value(body) + .map_err(|error| crate::params::Error::Serialize(Source::new(error)))? else { return Err(crate::params::Error::Body); }; @@ -183,9 +188,9 @@ mod tests { assert_eq!(arguments.get("enabled"), Some(&Value::Null)); assert_eq!(arguments.get("missing"), None); let invalid = serde_json::from_value(json!({"enabled":0})).unwrap(); - assert_eq!( - parse_options::(&invalid).err().unwrap().path, - "enabled" - ); + let error = parse_options::(&invalid).err().unwrap(); + assert_eq!(error.path, "enabled"); + let report = litellm_error::Report::new(&error).to_string(); + assert!(report.starts_with("invalid argument enabled: invalid type")); } } diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs index 162d325e4f4..b6f21f3ac16 100644 --- a/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs @@ -342,15 +342,15 @@ mod tests { #[case::request_timed_out("Request timed out")] #[case::timed_out_generating("Timed out generating response")] #[case::read_operation("The read operation timed out")] - fn timeout_markers_win_over_every_family(#[case] marker: &str) { + fn timeout_markers_win_over_every_family( + #[case] marker: &str, + #[values("mistral", "vertex_ai", "cohere", "reducto")] provider: &str, + ) { let body = format!("rate limit invalid api token {marker}"); - for provider in ["mistral", "vertex_ai", "cohere", "reducto"] { - assert_eq!( - mapped(provider, &http(429, &body)).error, - PublicError::Timeout { status: 408 }, - "{provider}" - ); - } + assert_eq!( + mapped(provider, &http(429, &body)).error, + PublicError::Timeout { status: 408 } + ); } #[test] diff --git a/litellm-rust/crates/core-utils/src/params.rs b/litellm-rust/crates/core-utils/src/params.rs index 9545a3ef17b..f163399b878 100644 --- a/litellm-rust/crates/core-utils/src/params.rs +++ b/litellm-rust/crates/core-utils/src/params.rs @@ -4,6 +4,8 @@ pub enum Error { ExtraBody, #[error("invalid request: body must be a JSON object")] Body, + #[error("invalid request: body could not be serialized")] + Serialize(#[source] litellm_error::Source), } use std::ops::{Deref, DerefMut}; diff --git a/litellm-rust/crates/core-utils/src/url_utils.rs b/litellm-rust/crates/core-utils/src/url_utils.rs index ebbe7fca00b..63e4f041a72 100644 --- a/litellm-rust/crates/core-utils/src/url_utils.rs +++ b/litellm-rust/crates/core-utils/src/url_utils.rs @@ -4,7 +4,7 @@ use url::Url; #[derive(Debug, thiserror::Error)] pub enum ApiUrlError { - #[error("invalid URL: {0}")] + #[error("invalid URL")] Parse(#[from] url::ParseError), #[error("URL cannot be used as a base")] CannotBeBase, diff --git a/litellm-rust/crates/cost/Cargo.toml b/litellm-rust/crates/cost/Cargo.toml index 3ed27e82c9a..6921e6151b5 100644 --- a/litellm-rust/crates/cost/Cargo.toml +++ b/litellm-rust/crates/cost/Cargo.toml @@ -5,6 +5,9 @@ edition.workspace = true license.workspace = true repository.workspace = true +[dependencies] +thiserror.workspace = true + [dev-dependencies] criterion.workspace = true proptest.workspace = true diff --git a/litellm-rust/crates/cost/src/error.rs b/litellm-rust/crates/cost/src/error.rs new file mode 100644 index 00000000000..1753080617f --- /dev/null +++ b/litellm-rust/crates/cost/src/error.rs @@ -0,0 +1,27 @@ +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum Error { + #[error("model has no input rate")] + MissingInputRate, + #[error("model has no output rate")] + MissingOutputRate, + #[error("rate must be finite and nonnegative")] + InvalidRate, + #[error("region multiplier must be finite and positive")] + InvalidRegionMultiplier, + #[error("off-peak pricing needs a billing minute within the day")] + InvalidBillingTime, + #[error("off-peak window must start before it ends within the day")] + InvalidOffPeakWindow, + #[error("cached tokens exceed prompt tokens")] + CacheExceedsPrompt, + #[error("cache write details do not add up to the cache write tokens")] + InvalidCacheWriteDetails, + #[error("token count or cost overflows")] + TokenCountOverflow, + #[error("service tier is repeated or cannot carry its own rates")] + DuplicateTier, + #[error("prompt token threshold is repeated")] + DuplicateThreshold, + #[error("service tier is repeated or cannot carry its own rates within a threshold")] + DuplicateThresholdTier, +} diff --git a/litellm-rust/crates/cost/src/lib.rs b/litellm-rust/crates/cost/src/lib.rs index 32cc755ce15..24c69afd9d4 100644 --- a/litellm-rust/crates/cost/src/lib.rs +++ b/litellm-rust/crates/cost/src/lib.rs @@ -1,3 +1,7 @@ +mod error; + +pub use error::Error; + #[derive(Clone, Copy, Debug, PartialEq)] pub enum Rate { Missing, @@ -156,22 +160,6 @@ pub struct EffectiveRates { pub cache_write_1h: f64, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum PricingError { - MissingInputRate, - MissingOutputRate, - InvalidRate, - InvalidRegionMultiplier, - InvalidBillingTime, - InvalidOffPeakWindow, - CacheExceedsPrompt, - InvalidCacheWriteDetails, - TokenCountOverflow, - DuplicateTier, - DuplicateThreshold, - DuplicateThresholdTier, -} - fn selected_tier(tier: ServiceTier) -> ServiceTier { if tier == ServiceTier::Fast { ServiceTier::Priority @@ -210,9 +198,9 @@ fn valid_rates(rates: Rates) -> bool { }) } -fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> { +fn validate_tiers(tiers: &[TierRates], duplicate: Error) -> Result<(), Error> { if tiers.iter().any(|entry| !valid_rates(entry.rates)) { - return Err(PricingError::InvalidRate); + return Err(Error::InvalidRate); } if tiers.iter().enumerate().any(|(index, entry)| { matches!( @@ -227,20 +215,20 @@ fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), Pr Ok(()) } -pub fn compile(pricing: &Pricing<'_>) -> Result { +pub fn compile(pricing: &Pricing<'_>) -> Result { if !valid_rates(pricing.standard) { - return Err(PricingError::InvalidRate); + return Err(Error::InvalidRate); } - validate_tiers(pricing.tiers, PricingError::DuplicateTier)?; + validate_tiers(pricing.tiers, Error::DuplicateTier)?; if let Some(window) = pricing.off_peak { if window.start_utc_minute >= 1440 || window.end_utc_minute > 1440 || window.start_utc_minute >= window.end_utc_minute { - return Err(PricingError::InvalidOffPeakWindow); + return Err(Error::InvalidOffPeakWindow); } if !valid_rates(window.rates) { - return Err(PricingError::InvalidRate); + return Err(Error::InvalidRate); } } let mut thresholds: Vec<_> = pricing @@ -248,9 +236,9 @@ pub fn compile(pricing: &Pricing<'_>) -> Result { .iter() .map(|entry| { if !valid_rates(entry.standard) { - return Err(PricingError::InvalidRate); + return Err(Error::InvalidRate); } - validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?; + validate_tiers(entry.tiers, Error::DuplicateThresholdTier)?; Ok(CompiledThreshold { above_prompt_tokens: entry.above_prompt_tokens, standard: entry.standard, @@ -263,7 +251,7 @@ pub fn compile(pricing: &Pricing<'_>) -> Result { .windows(2) .any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens) { - return Err(PricingError::DuplicateThreshold); + return Err(Error::DuplicateThreshold); } Ok(PricingPlan { standard: pricing.standard, @@ -274,11 +262,7 @@ pub fn compile(pricing: &Pricing<'_>) -> Result { } impl PricingPlan { - fn resolve_rates( - &self, - request: &Request, - threshold_tokens: u64, - ) -> Result { + fn resolve_rates(&self, request: &Request, threshold_tokens: u64) -> Result { let tier = selected_tier(request.service_tier); let base = self .tiers @@ -305,13 +289,13 @@ impl PricingPlan { || window.end_utc_minute > 1440 || window.start_utc_minute >= window.end_utc_minute { - return Err(PricingError::InvalidOffPeakWindow); + return Err(Error::InvalidOffPeakWindow); } let minute = request .billed_at_utc_minute - .ok_or(PricingError::InvalidBillingTime)?; + .ok_or(Error::InvalidBillingTime)?; if minute >= 1440 { - return Err(PricingError::InvalidBillingTime); + return Err(Error::InvalidBillingTime); } if (window.start_utc_minute..window.end_utc_minute).contains(&minute) { Ok(window.rates.overlay(selected)) @@ -322,26 +306,26 @@ impl PricingPlan { } } - fn checked_rate(rate: Rate, missing: PricingError) -> Result { + fn checked_rate(rate: Rate, missing: Error) -> Result { let value = rate.value().ok_or(missing)?; if !value.is_finite() || value < 0.0 { - return Err(PricingError::InvalidRate); + return Err(Error::InvalidRate); } Ok(value) } - pub fn calculate(&self, request: &Request) -> Result { + pub fn calculate(&self, request: &Request) -> Result { let usage = request.usage; let cached = usage .cache_read_tokens .checked_add(usage.cache_write_tokens) - .ok_or(PricingError::TokenCountOverflow)?; + .ok_or(Error::TokenCountOverflow)?; let (regular, threshold_tokens) = match usage.prompt_convention { PromptConvention::IncludesCache => ( usage .prompt_tokens .checked_sub(cached) - .ok_or(PricingError::CacheExceedsPrompt)?, + .ok_or(Error::CacheExceedsPrompt)?, usage.prompt_tokens, ), PromptConvention::ExcludesCache => ( @@ -349,7 +333,7 @@ impl PricingPlan { usage .prompt_tokens .checked_add(cached) - .ok_or(PricingError::TokenCountOverflow)?, + .ok_or(Error::TokenCountOverflow)?, ), }; let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) { @@ -357,26 +341,20 @@ impl PricingPlan { (Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => { (five, one) } - _ => return Err(PricingError::InvalidCacheWriteDetails), + _ => return Err(Error::InvalidCacheWriteDetails), }; let rates = self.resolve_rates(request, threshold_tokens)?; - let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?; - let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?; - let read = Self::checked_rate( - rates.cache_read.or(rates.input), - PricingError::MissingInputRate, - )?; - let write = Self::checked_rate( - rates.cache_write.or(rates.input), - PricingError::MissingInputRate, - )?; + let input = Self::checked_rate(rates.input, Error::MissingInputRate)?; + let output = Self::checked_rate(rates.output, Error::MissingOutputRate)?; + let read = Self::checked_rate(rates.cache_read.or(rates.input), Error::MissingInputRate)?; + let write = Self::checked_rate(rates.cache_write.or(rates.input), Error::MissingInputRate)?; let write_1h = Self::checked_rate( rates.cache_write_1h.or(rates.cache_write).or(rates.input), - PricingError::MissingInputRate, + Error::MissingInputRate, )?; let multiplier = request.region_multiplier.unwrap_or(1.0); if !multiplier.is_finite() || multiplier <= 0.0 { - return Err(PricingError::InvalidRegionMultiplier); + return Err(Error::InvalidRegionMultiplier); } let cost = Cost { uncached_input: regular as f64 * input, @@ -394,12 +372,12 @@ impl PricingPlan { }, }; if !cost.total().is_finite() { - return Err(PricingError::TokenCountOverflow); + return Err(Error::TokenCountOverflow); } Ok(cost) } } -pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result { +pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result { compile(pricing)?.calculate(request) } diff --git a/litellm-rust/crates/cost/tests/calculation.rs b/litellm-rust/crates/cost/tests/calculation.rs index f8152483a7c..af1a73d8c61 100644 --- a/litellm-rust/crates/cost/tests/calculation.rs +++ b/litellm-rust/crates/cost/tests/calculation.rs @@ -1,5 +1,5 @@ use litellm_cost::{ - OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier, + Error, OffPeakRates, Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile, }; use rstest::rstest; @@ -132,7 +132,7 @@ fn split_writes_and_invalid_accounting() { }; assert_eq!( calculate(&pricing(standard), &overlapping), - Err(PricingError::CacheExceedsPrompt) + Err(Error::CacheExceedsPrompt) ); let incomplete = Request { usage: Usage { @@ -143,7 +143,7 @@ fn split_writes_and_invalid_accounting() { }; assert_eq!( calculate(&pricing(standard), &incomplete), - Err(PricingError::InvalidCacheWriteDetails) + Err(Error::InvalidCacheWriteDetails) ); } @@ -297,10 +297,10 @@ fn compile_rejects_ambiguous_rates() { }; assert_eq!( compile(&specification).err(), - Some(PricingError::DuplicateThreshold) + Some(Error::DuplicateThreshold) ); let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0))); - assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate)); + assert_eq!(compile(&invalid).err(), Some(Error::InvalidRate)); } #[test] @@ -326,7 +326,7 @@ fn off_peak_is_one_non_wrapping_utc_window() { }; assert_eq!( calculate(&specification, &base), - Err(PricingError::InvalidBillingTime) + Err(Error::InvalidBillingTime) ); assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0); assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0); @@ -347,11 +347,11 @@ fn missing_rates_and_free_rates_remain_distinct() { }; assert_eq!( calculate(&pricing(Rates::EMPTY), &empty), - Err(PricingError::MissingInputRate) + Err(Error::MissingInputRate) ); assert_eq!( calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty), - Err(PricingError::MissingOutputRate) + Err(Error::MissingOutputRate) ); assert_eq!( calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty) @@ -481,7 +481,7 @@ fn regional_multiplier_applies_after_input_components_are_summed() { }; assert_eq!( calculate(&pricing(standard), &invalid), - Err(PricingError::InvalidRegionMultiplier) + Err(Error::InvalidRegionMultiplier) ); } @@ -531,5 +531,5 @@ fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() { thresholds: &[], ..specification }; - assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier)); + assert_eq!(compile(&invalid).err(), Some(Error::DuplicateTier)); } diff --git a/litellm-rust/crates/error/src/lib.rs b/litellm-rust/crates/error/src/lib.rs index ce16ed3846a..ec8d5c3e1f4 100644 --- a/litellm-rust/crates/error/src/lib.rs +++ b/litellm-rust/crates/error/src/lib.rs @@ -16,6 +16,14 @@ impl fmt::Display for Report<'_> { } } +pub fn find<'a, T: Error + 'static>(error: &'a (dyn Error + 'static)) -> Option<&'a T> { + std::iter::successors(Some(error), |&cause| cause.source()).find_map(|cause| { + cause + .downcast_ref::() + .or_else(|| cause.downcast_ref::()?.downcast_ref::()) + }) +} + #[derive(Clone, Debug, thiserror::Error)] #[error(transparent)] pub struct Source(Arc); @@ -26,6 +34,12 @@ impl Source { } } +impl From> for Source { + fn from(error: Box) -> Self { + Self(Arc::from(error)) + } +} + impl Deref for Source { type Target = dyn Error + Send + Sync; @@ -48,7 +62,7 @@ mod tests { use rstest::rstest; - use super::{Report, Source}; + use super::{Report, Source, find}; #[derive(Debug, thiserror::Error)] enum Leaf { @@ -90,13 +104,25 @@ mod tests { assert_eq!(Report::new(&error).to_string(), expected); } - #[test] - fn a_source_still_downcasts_to_the_original_cause() { - let source = Source::new(denied()); - let cause = source.downcast_ref::().expect("io cause"); + #[rstest] + #[case::direct_source(Outer::Startup(Leaf::Read(denied())))] + #[case::behind_an_erased_source(Outer::Startup(Leaf::Parse(Source::new(Leaf::Read( + denied() + )))))] + #[case::behind_a_boxed_source(Outer::Startup(Leaf::Parse(Source::from( + Box::::from(denied()) + ))))] + #[case::behind_a_transparent_layer(Outer::Nested(Leaf::Parse(Source::new(denied()))))] + fn find_reaches_the_typed_cause_anywhere_in_the_chain(#[case] error: Outer) { + let cause = find::(&error).expect("io cause"); assert_eq!(cause.kind(), io::ErrorKind::PermissionDenied); } + #[test] + fn find_answers_none_when_the_chain_has_no_such_cause() { + assert!(find::(&Outer::Nested(Leaf::Empty)).is_none()); + } + #[test] fn a_clone_shares_its_cause() { let source = Source::new(denied()); diff --git a/litellm-rust/crates/framer/src/error.rs b/litellm-rust/crates/framer/src/error.rs index 879d7557671..dde421d71af 100644 --- a/litellm-rust/crates/framer/src/error.rs +++ b/litellm-rust/crates/framer/src/error.rs @@ -1,21 +1,21 @@ #[cfg(feature = "sse")] #[derive(Debug, thiserror::Error)] pub enum SseError { - #[error("body stream failed: {0}")] + #[error("body stream failed")] Body(#[from] std::io::Error), - #[error("SSE field is not UTF-8: {0}")] + #[error("SSE field is not UTF-8")] InvalidUtf8(#[from] std::str::Utf8Error), } #[cfg(feature = "aws")] #[derive(Debug, thiserror::Error)] pub enum EventStreamError { - #[error("body stream failed: {0}")] + #[error("body stream failed")] Body(#[from] std::io::Error), #[error("invalid AWS EventStream frame length: {0}")] InvalidLength(usize), #[error("truncated AWS EventStream frame")] Truncated, - #[error("malformed AWS EventStream frame: {0}")] + #[error("malformed AWS EventStream frame")] Malformed(#[from] aws_smithy_eventstream::error::Error), } diff --git a/litellm-rust/crates/framer/src/framer.rs b/litellm-rust/crates/framer/src/framer.rs deleted file mode 100644 index 507aed54700..00000000000 --- a/litellm-rust/crates/framer/src/framer.rs +++ /dev/null @@ -1,13 +0,0 @@ -use futures_util::Stream; - -use crate::Error; - -pub trait Framer: Send { - type Frame: Send; - - fn frame(self, input: S) -> impl Stream> + Send - where - S: Stream> + Send, - B: bytes::Buf + Send, - E: std::error::Error + Send + Sync + 'static; -} diff --git a/litellm-rust/crates/gateway-auth/Cargo.toml b/litellm-rust/crates/gateway-auth/Cargo.toml index cf040c6eef3..13e7380c102 100644 --- a/litellm-rust/crates/gateway-auth/Cargo.toml +++ b/litellm-rust/crates/gateway-auth/Cargo.toml @@ -20,6 +20,7 @@ veil.workspace = true [dev-dependencies] futures-util.workspace = true +litellm-error.workspace = true rstest.workspace = true tokio.workspace = true tower = { version = "0.5.3", features = ["util"] } diff --git a/litellm-rust/crates/gateway-auth/tests/auth.rs b/litellm-rust/crates/gateway-auth/tests/auth.rs index 5a2a3ded834..69170eb9c94 100644 --- a/litellm-rust/crates/gateway-auth/tests/auth.rs +++ b/litellm-rust/crates/gateway-auth/tests/auth.rs @@ -26,9 +26,9 @@ impl SecretSource for Secrets { match name { "MASTER_KEY" => Ok(Some(SecretValue::new("resolved-key"))), "EMPTY" => Ok(Some(SecretValue::new(""))), - "ERROR" => Err(litellm_secrets::Error::ExternalRead(Box::new( - std::io::Error::other("private-backend-detail"), - ))), + "ERROR" => Err(litellm_secrets::Error::ExternalRead( + litellm_error::Source::new(std::io::Error::other("private-backend-detail")), + )), _ => Ok(None), } }) diff --git a/litellm-rust/crates/host-python/Cargo.toml b/litellm-rust/crates/host-python/Cargo.toml index 8d220b19def..0b0107d0548 100644 --- a/litellm-rust/crates/host-python/Cargo.toml +++ b/litellm-rust/crates/host-python/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] litellm-coroutine.workspace = true +litellm-error.workspace = true litellm-host.workspace = true bytes.workspace = true diff --git a/litellm-rust/crates/host-python/src/binding.rs b/litellm-rust/crates/host-python/src/binding.rs index 91fbc8c0e52..e36b7d70c28 100644 --- a/litellm-rust/crates/host-python/src/binding.rs +++ b/litellm-rust/crates/host-python/src/binding.rs @@ -5,7 +5,7 @@ use pyo3::types::PyDict; /// Converts requests, responses, stream values and errors at the Python boundary. pub trait PythonBinding: PythonOwned { - type Protocol: Protocol; + type Protocol: Protocol; /// The public exception a native failure maps to, kept as a value until the driver /// raises it. diff --git a/litellm-rust/crates/host-python/src/driver.rs b/litellm-rust/crates/host-python/src/driver.rs index f8e35aa09ba..5dd9a72eb4a 100644 --- a/litellm-rust/crates/host-python/src/driver.rs +++ b/litellm-rust/crates/host-python/src/driver.rs @@ -112,7 +112,7 @@ enum Setup { /// The route's public exception for a native failure. When classification itself fails, /// that failure is raised with the native error's text as its `__context__`. fn classified(py: Python<'_>, binding: &H, error: ErrorOf) -> PyErr { - let native = error.to_string(); + let native = litellm_error::Report::new(&error).to_string(); let classifier_error = match binding.map_error(py, error) { Ok(failure) => return failure.into(), Err(classifier_error) => classifier_error, diff --git a/litellm-rust/crates/host-python/src/error.rs b/litellm-rust/crates/host-python/src/error.rs index b06a4a2fd41..1c4c53e68e6 100644 --- a/litellm-rust/crates/host-python/src/error.rs +++ b/litellm-rust/crates/host-python/src/error.rs @@ -1,3 +1,4 @@ +use litellm_error::Report; use pyo3::{ PyErr, exceptions::{PyRuntimeError, PyTypeError}, @@ -30,10 +31,9 @@ pub enum Error { impl From for PyErr { fn from(error: Error) -> Self { + let message = Report::new(&error).to_string(); match error { - Error::SendBeforeStart | Error::FileContentType { .. } => { - PyTypeError::new_err(error.to_string()) - } + Error::SendBeforeStart | Error::FileContentType { .. } => PyTypeError::new_err(message), Error::MissingState | Error::ExecutionRunning | Error::ExecutionClosed @@ -41,7 +41,7 @@ impl From for PyErr { | Error::ExecutionClosedWhileRunning | Error::NativeExecutionClosed | Error::SyncInsideTokio - | Error::ContextBodyProducedNothing => PyRuntimeError::new_err(error.to_string()), + | Error::ContextBodyProducedNothing => PyRuntimeError::new_err(message), } } } diff --git a/litellm-rust/crates/host-python/src/marshal.rs b/litellm-rust/crates/host-python/src/marshal.rs index ae9066379ae..465521cf9d2 100644 --- a/litellm-rust/crates/host-python/src/marshal.rs +++ b/litellm-rust/crates/host-python/src/marshal.rs @@ -14,7 +14,8 @@ pub fn from_py_argument(value: &Bound<'_, PyAny>) -> PyResult where T: DeserializeOwned, { - pythonize::depythonize(value).map_err(|error| PyValueError::new_err(error.to_string())) + pythonize::depythonize(value) + .map_err(|error| PyValueError::new_err(litellm_error::Report::new(&error).to_string())) } pub fn from_py(value: &Bound<'_, PyAny>) -> PyResult diff --git a/litellm-rust/crates/host-python/tests/support/mod.rs b/litellm-rust/crates/host-python/tests/support/mod.rs index ef09135698e..003fce74309 100644 --- a/litellm-rust/crates/host-python/tests/support/mod.rs +++ b/litellm-rust/crates/host-python/tests/support/mod.rs @@ -92,6 +92,8 @@ impl std::fmt::Display for Error { } } +impl std::error::Error for Error {} + impl From for Error { fn from(fault: MachineFault) -> Self { Self(format!("{fault:?}")) diff --git a/litellm-rust/crates/http/Cargo.toml b/litellm-rust/crates/http/Cargo.toml index dbdb126ec94..1f130f44ab1 100644 --- a/litellm-rust/crates/http/Cargo.toml +++ b/litellm-rust/crates/http/Cargo.toml @@ -11,6 +11,7 @@ test-support = [] [dependencies] http.workspace = true litellm-core-utils.workspace = true +litellm-error.workspace = true hyper-util.workspace = true reqwest.workspace = true rustls.workspace = true diff --git a/litellm-rust/crates/http/src/error.rs b/litellm-rust/crates/http/src/error.rs index 220ea8d9858..fb6d6f7a216 100644 --- a/litellm-rust/crates/http/src/error.rs +++ b/litellm-rust/crates/http/src/error.rs @@ -1,7 +1,11 @@ -use std::{fmt, io, path::PathBuf, sync::Arc}; +use std::{io, path::PathBuf, sync::Arc}; +use tokio::time::error::Elapsed; use tokio_tungstenite::tungstenite; +const UPSTREAM_BODY_MAX_CHARS: usize = 256; +const TRUNCATION_MARKER: &str = "... (truncated)"; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TlsSource { CaBundle, @@ -14,60 +18,64 @@ pub struct Error(Arc); #[derive(Debug, thiserror::Error)] pub(crate) enum Repr { - #[error("could not read {}: {cause}", path.display())] + #[error("could not read {}", path.display())] Read { path: PathBuf, tls_source: TlsSource, - cause: io::Error, + source: io::Error, }, - #[error("{} is not a PEM file: {cause}", path.display())] + #[error("could not load {}", path.display())] InvalidPem { path: PathBuf, tls_source: TlsSource, - cause: PemFault, + source: PemFault, }, - #[error("could not build the HTTP client: {}", Chain(.0))] - Client(reqwest::Error), - #[error("could not build the HTTP client: {0}")] - TlsConfig(rustls::Error), + #[error("could not build the HTTP client")] + Client(#[source] reqwest::Error), + #[error("could not build the TLS configuration")] + TlsConfig(#[source] rustls::Error), - #[error("request body could not be serialized: {0}")] - RequestBody(serde_json::Error), - #[error("invalid request: {0}")] - Request(http::Error), - #[error("invalid request: {}", Chain(.0))] - RequestUrl(reqwest::Error), - #[error("request forwards a header the signer computes: {0}")] + #[error("request body could not be serialized")] + RequestBody(#[source] serde_json::Error), + #[error("invalid request")] + Request(#[source] http::Error), + #[error("invalid request")] + RequestUrl(#[source] reqwest::Error), + #[error("request forwards the header {0:?}, which the signer computes")] ComputedHeader(String), #[error("request signing failed: a streaming body cannot be signed")] UnsignableBody, - #[error("request signing failed: {0}")] - Signature(Box), + #[error("request forwards a header that cannot be signed")] + UnsignableHeader(#[source] litellm_error::Source), + #[error("request signing failed")] + Signature(#[source] litellm_error::Source), - #[error("could not reach the provider: {}", Chain(.0))] - Connect(reqwest::Error), - #[error("upstream network error: {}", Chain(.0))] - Network(reqwest::Error), - #[error("upstream request timed out: {}", Chain(.0))] - Timeout(reqwest::Error), + #[error("could not reach the provider")] + Connect(#[source] reqwest::Error), + #[error("upstream network error")] + Network(#[source] reqwest::Error), + #[error("upstream request timed out")] + Timeout(#[source] reqwest::Error), #[error("upstream request failed with status {status}: {body}")] Status { status: u16, body: String }, - #[error("invalid WebSocket URL: {0}")] - WebSocketUrl(url::ParseError), + #[error("invalid WebSocket URL")] + WebSocketUrl(#[source] url::ParseError), #[error("unsupported WebSocket URL scheme {0:?}")] WebSocketScheme(String), + #[error("could not rewrite the WebSocket URL scheme to {0}")] + WebSocketSchemeRewrite(&'static str), #[error("WebSocket handshake returned {0}")] Handshake(HandshakeFault), #[error("WebSocket connection timed out")] - HandshakeTimeout, - #[error("upstream WebSocket error: {0}")] - WebSocket(tungstenite::Error), + HandshakeTimeout(#[source] Elapsed), + #[error("upstream WebSocket error")] + WebSocket(#[source] tungstenite::Error), #[error("media URL rejected by network policy")] BlockedUrl, - #[error("could not resolve the media host: {0}")] - Resolve(io::Error), + #[error("could not resolve the media host")] + Resolve(#[source] io::Error), #[error("media download is disabled")] DownloadDisabled, #[error("media download exceeds the maximum size")] @@ -77,11 +85,13 @@ pub(crate) enum Repr { #[error("media redirect is missing a Location header")] MissingRedirectLocation, #[error("invalid media redirect")] - InvalidRedirect, + InvalidRedirect(#[source] RedirectFault), #[error("media download failed with status {0}")] DownloadStatus(u16), + #[error("media download failed")] + Download(#[source] reqwest::Error), #[error("media download timed out")] - DownloadTimeout, + DownloadTimeout(#[source] Elapsed), } #[derive(Debug, thiserror::Error)] @@ -100,25 +110,33 @@ pub(crate) enum HandshakeFault { pub(crate) enum PemFault { #[error("no certificates found")] NoCertificates, - #[error(transparent)] + #[error("could not parse PEM")] Parse(#[from] rustls::pki_types::pem::Error), - #[error(transparent)] + #[error("rustls rejected the certificate or key")] Rejected(#[from] rustls::Error), } -struct Chain<'a>(&'a reqwest::Error); +#[derive(Debug, thiserror::Error)] +pub(crate) enum RedirectFault { + #[error("Location header is not valid text")] + Location(#[from] http::header::ToStrError), + #[error("Location is not a valid URL")] + Url(#[from] url::ParseError), +} -impl fmt::Display for Chain<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "{}", self.0)?; - std::iter::successors(std::error::Error::source(self.0), |cause| cause.source()) - .try_for_each(|cause| write!(formatter, ": {cause}")) +impl Repr { + pub(crate) fn download(error: reqwest::Error) -> Self { + Self::Download(error.without_url()) } } impl Error { pub fn upstream(status: u16, body: String) -> Self { - Repr::Status { status, body }.into() + Repr::Status { + status, + body: bounded(body), + } + .into() } pub fn computed_header(name: impl Into) -> Self { @@ -129,69 +147,299 @@ impl Error { Repr::UnsignableBody.into() } + pub fn unsignable_header(cause: impl std::error::Error + Send + Sync + 'static) -> Self { + Repr::UnsignableHeader(litellm_error::Source::new(cause)).into() + } + pub fn signature(cause: impl std::error::Error + Send + Sync + 'static) -> Self { - Repr::Signature(Box::new(cause)).into() + Repr::Signature(litellm_error::Source::new(cause)).into() } /// The request may have reached the provider, so retrying elsewhere could /// repeat a billed call. pub fn dispatched(&self) -> bool { - matches!( - *self.0, + match &*self.0 { Repr::Network(_) - | Repr::Timeout(_) - | Repr::Status { .. } - | Repr::Handshake(_) - | Repr::HandshakeTimeout - | Repr::WebSocket(_) - ) + | Repr::Timeout(_) + | Repr::Status { .. } + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) => true, + Repr::Read { .. } + | Repr::InvalidPem { .. } + | Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::Request(_) + | Repr::RequestUrl(_) + | Repr::ComputedHeader(_) + | Repr::UnsignableBody + | Repr::UnsignableHeader(_) + | Repr::Signature(_) + | Repr::Connect(_) + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::BlockedUrl + | Repr::Resolve(_) + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects + | Repr::MissingRedirectLocation + | Repr::InvalidRedirect(_) + | Repr::DownloadStatus(_) + | Repr::Download(_) + | Repr::DownloadTimeout(_) => false, + } } pub fn is_request(&self) -> bool { - matches!( - *self.0, + match &*self.0 { Repr::Request(_) - | Repr::RequestUrl(_) - | Repr::ComputedHeader(_) - | Repr::WebSocketUrl(_) - | Repr::WebSocketScheme(_) - | Repr::BlockedUrl - | Repr::DownloadDisabled - | Repr::DownloadTooLarge - | Repr::TooManyRedirects - ) + | Repr::RequestUrl(_) + | Repr::ComputedHeader(_) + | Repr::UnsignableHeader(_) + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::BlockedUrl + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects => true, + Repr::Read { .. } + | Repr::InvalidPem { .. } + | Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::UnsignableBody + | Repr::Signature(_) + | Repr::Connect(_) + | Repr::Network(_) + | Repr::Timeout(_) + | Repr::Status { .. } + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) + | Repr::Resolve(_) + | Repr::MissingRedirectLocation + | Repr::InvalidRedirect(_) + | Repr::DownloadStatus(_) + | Repr::Download(_) + | Repr::DownloadTimeout(_) => false, + } } pub fn is_response(&self) -> bool { - matches!( - *self.0, - Repr::MissingRedirectLocation | Repr::InvalidRedirect - ) + match &*self.0 { + Repr::MissingRedirectLocation | Repr::InvalidRedirect(_) => true, + Repr::Read { .. } + | Repr::InvalidPem { .. } + | Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::Request(_) + | Repr::RequestUrl(_) + | Repr::ComputedHeader(_) + | Repr::UnsignableBody + | Repr::UnsignableHeader(_) + | Repr::Signature(_) + | Repr::Connect(_) + | Repr::Network(_) + | Repr::Timeout(_) + | Repr::Status { .. } + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) + | Repr::BlockedUrl + | Repr::Resolve(_) + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects + | Repr::DownloadStatus(_) + | Repr::Download(_) + | Repr::DownloadTimeout(_) => false, + } } pub fn is_computed_header(&self) -> bool { - matches!(*self.0, Repr::ComputedHeader(_)) + match &*self.0 { + Repr::ComputedHeader(_) => true, + Repr::Read { .. } + | Repr::InvalidPem { .. } + | Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::Request(_) + | Repr::RequestUrl(_) + | Repr::UnsignableBody + | Repr::UnsignableHeader(_) + | Repr::Signature(_) + | Repr::Connect(_) + | Repr::Network(_) + | Repr::Timeout(_) + | Repr::Status { .. } + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) + | Repr::BlockedUrl + | Repr::Resolve(_) + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects + | Repr::MissingRedirectLocation + | Repr::InvalidRedirect(_) + | Repr::DownloadStatus(_) + | Repr::Download(_) + | Repr::DownloadTimeout(_) => false, + } + } + + pub fn is_connect(&self) -> bool { + match &*self.0 { + Repr::Connect(_) => true, + Repr::ComputedHeader(_) + | Repr::Read { .. } + | Repr::InvalidPem { .. } + | Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::Request(_) + | Repr::RequestUrl(_) + | Repr::UnsignableBody + | Repr::UnsignableHeader(_) + | Repr::Signature(_) + | Repr::Network(_) + | Repr::Timeout(_) + | Repr::Status { .. } + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) + | Repr::BlockedUrl + | Repr::Resolve(_) + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects + | Repr::MissingRedirectLocation + | Repr::InvalidRedirect(_) + | Repr::DownloadStatus(_) + | Repr::Download(_) + | Repr::DownloadTimeout(_) => false, + } } pub fn upstream_response(&self) -> Option<(u16, &str)> { match &*self.0 { Repr::Status { status, body } => Some((*status, body)), - _ => None, + Repr::Read { .. } + | Repr::InvalidPem { .. } + | Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::Request(_) + | Repr::RequestUrl(_) + | Repr::ComputedHeader(_) + | Repr::UnsignableBody + | Repr::UnsignableHeader(_) + | Repr::Signature(_) + | Repr::Connect(_) + | Repr::Network(_) + | Repr::Timeout(_) + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) + | Repr::BlockedUrl + | Repr::Resolve(_) + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects + | Repr::MissingRedirectLocation + | Repr::InvalidRedirect(_) + | Repr::DownloadStatus(_) + | Repr::Download(_) + | Repr::DownloadTimeout(_) => None, } } pub fn status(&self) -> Option { - match *self.0 { - Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(status), - Repr::Timeout(_) | Repr::DownloadTimeout => Some(408), - _ => None, + match &*self.0 { + Repr::Status { status, .. } | Repr::DownloadStatus(status) => Some(*status), + Repr::Timeout(_) | Repr::DownloadTimeout(_) => Some(408), + Repr::Read { .. } + | Repr::InvalidPem { .. } + | Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::Request(_) + | Repr::RequestUrl(_) + | Repr::ComputedHeader(_) + | Repr::UnsignableBody + | Repr::UnsignableHeader(_) + | Repr::Signature(_) + | Repr::Connect(_) + | Repr::Network(_) + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) + | Repr::BlockedUrl + | Repr::Resolve(_) + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects + | Repr::MissingRedirectLocation + | Repr::InvalidRedirect(_) + | Repr::Download(_) => None, } } pub fn tls_source(&self) -> Option { - match *self.0 { - Repr::Read { tls_source, .. } | Repr::InvalidPem { tls_source, .. } => Some(tls_source), - _ => None, + match &*self.0 { + Repr::Read { tls_source, .. } | Repr::InvalidPem { tls_source, .. } => { + Some(*tls_source) + } + Repr::Client(_) + | Repr::TlsConfig(_) + | Repr::RequestBody(_) + | Repr::Request(_) + | Repr::RequestUrl(_) + | Repr::ComputedHeader(_) + | Repr::UnsignableBody + | Repr::UnsignableHeader(_) + | Repr::Signature(_) + | Repr::Connect(_) + | Repr::Network(_) + | Repr::Timeout(_) + | Repr::Status { .. } + | Repr::WebSocketUrl(_) + | Repr::WebSocketScheme(_) + | Repr::WebSocketSchemeRewrite(_) + | Repr::Handshake(_) + | Repr::HandshakeTimeout(_) + | Repr::WebSocket(_) + | Repr::BlockedUrl + | Repr::Resolve(_) + | Repr::DownloadDisabled + | Repr::DownloadTooLarge + | Repr::TooManyRedirects + | Repr::MissingRedirectLocation + | Repr::InvalidRedirect(_) + | Repr::DownloadStatus(_) + | Repr::Download(_) + | Repr::DownloadTimeout(_) => None, } } @@ -201,6 +449,13 @@ impl Error { } } +fn bounded(body: String) -> String { + match body.char_indices().nth(UPSTREAM_BODY_MAX_CHARS) { + None => body, + Some((cut, _)) => format!("{}{TRUNCATION_MARKER}", &body[..cut]), + } +} + impl PartialEq for Error { fn eq(&self, other: &Self) -> bool { Arc::ptr_eq(&self.0, &other.0) @@ -236,6 +491,11 @@ impl From for Error { #[cfg(test)] mod tests { + use litellm_error::Report; + use rstest::rstest; + + use super::{TRUNCATION_MARKER, UPSTREAM_BODY_MAX_CHARS}; + #[tokio::test] async fn transport_errors_remove_urls_and_keep_dispatch_context() { let error = reqwest::Client::builder() @@ -247,20 +507,20 @@ mod tests { .await .expect_err("invalid port"); let error = crate::Error::from(error); + let report = Report::new(&error).to_string(); assert!(!error.dispatched()); - assert!(!error.to_string().contains("secret")); - assert!(!error.to_string().contains("private")); + assert!(!report.contains("secret"), "{report}"); + assert!(!report.contains("private"), "{report}"); } - fn root_cause(error: &dyn std::error::Error) -> Option { - match error.source() { - Some(cause) => root_cause(cause).or_else(|| Some(cause.to_string())), - None => None, - } + fn root_cause<'a>( + error: &'a (dyn std::error::Error + 'static), + ) -> &'a (dyn std::error::Error + 'static) { + error.source().map_or(error, root_cause) } #[tokio::test] - async fn network_error_message_names_the_underlying_cause() { + async fn connect_failure_reports_its_root_cause_once() { let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); let address = listener.local_addr().expect("address"); drop(listener); @@ -272,10 +532,14 @@ mod tests { .send() .await .expect_err("nothing listens on the port"); - let root_cause = root_cause(&error).expect("reqwest reports a cause"); - let message = crate::Error::from(error).to_string(); - assert!(message.contains(&root_cause), "{message}"); - assert!(!message.contains("secret")); + let error = crate::Error::from(error); + let root_cause = root_cause(&error).to_string(); + let report = Report::new(&error).to_string(); + assert_eq!(report.matches(&root_cause).count(), 1, "{report}"); + assert!(!error.to_string().contains(&root_cause)); + assert!(error.is_connect()); + assert!(!report.contains("secret"), "{report}"); + assert!(!report.contains("private"), "{report}"); } #[tokio::test] @@ -303,6 +567,22 @@ mod tests { assert!(error.is_timeout()); let error = crate::Error::from(error); assert!(error.dispatched()); + assert!(!error.is_connect()); assert_eq!(error.status(), Some(408)); } + + #[rstest] + #[case::short("slow down".to_owned(), "slow down".to_owned())] + #[case::at_the_limit("\u{00e9}".repeat(UPSTREAM_BODY_MAX_CHARS), "\u{00e9}".repeat(UPSTREAM_BODY_MAX_CHARS))] + #[case::over_the_limit( + "\u{00e9}".repeat(UPSTREAM_BODY_MAX_CHARS + 1), + format!("{}{TRUNCATION_MARKER}", "\u{00e9}".repeat(UPSTREAM_BODY_MAX_CHARS)) + )] + fn upstream_bounds_the_provider_body_by_characters( + #[case] body: String, + #[case] expected: String, + ) { + let error = crate::Error::upstream(429, body); + assert_eq!(error.upstream_response(), Some((429, expected.as_str()))); + } } diff --git a/litellm-rust/crates/http/src/media.rs b/litellm-rust/crates/http/src/media.rs index a69cb75793d..4acaca5e502 100644 --- a/litellm-rust/crates/http/src/media.rs +++ b/litellm-rust/crates/http/src/media.rs @@ -149,7 +149,7 @@ impl MediaFetcher { } tokio::time::timeout(policy.timeout, self.fetch_before_deadline(url, policy)) .await - .map_err(|_| Repr::DownloadTimeout)? + .map_err(Repr::DownloadTimeout)? } async fn fetch_before_deadline( @@ -159,7 +159,13 @@ impl MediaFetcher { ) -> Result { let mut redirects_followed = 0; loop { - let mut response = self.client_for(&url).await?.get(url.clone()).send().await?; + let mut response = self + .client_for(&url) + .await? + .get(url.clone()) + .send() + .await + .map_err(Repr::download)?; if response.status().is_redirection() { if redirects_followed == policy.max_redirects { return Err(Repr::TooManyRedirects.into()); @@ -167,9 +173,12 @@ impl MediaFetcher { let location = response .headers() .get(reqwest::header::LOCATION) - .and_then(|value| value.to_str().ok()) - .ok_or(Repr::MissingRedirectLocation)?; - url = url.join(location).map_err(|_| Repr::InvalidRedirect)?; + .ok_or(Repr::MissingRedirectLocation)? + .to_str() + .map_err(|error| Repr::InvalidRedirect(error.into()))?; + url = url + .join(location) + .map_err(|error| Repr::InvalidRedirect(error.into()))?; redirects_followed += 1; continue; } @@ -187,7 +196,7 @@ impl MediaFetcher { .unwrap_or("application/octet-stream") .to_string(); let mut bytes = Vec::new(); - while let Some(chunk) = response.chunk().await? { + while let Some(chunk) = response.chunk().await.map_err(Repr::download)? { enforce_download_size(bytes.len() as u64 + chunk.len() as u64, policy.max_bytes)?; bytes.extend_from_slice(&chunk); } @@ -590,7 +599,27 @@ mod tests { .await .expect_err("fetch times out"); server.await.expect("server completes"); - assert!(matches!(error.repr(), Repr::DownloadTimeout)); + assert!(matches!(error.repr(), Repr::DownloadTimeout(_))); + assert!(!error.dispatched()); + } + + #[rstest::rstest] + #[case::closed_before_answering(b"")] + #[case::reset_mid_body(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nab")] + #[tokio::test] + async fn a_failed_download_never_reached_the_provider(#[case] response: &'static [u8]) { + let (url, server) = serve(response).await; + let url = url.join("?sig=secret").expect("valid test URL"); + let error = MediaFetcher::for_test(Client::for_test()) + .fetch(url, policy(10, 0)) + .await + .expect_err("download fails"); + server.await.expect("server completes"); + let report = litellm_error::Report::new(&error).to_string(); + assert!(!error.dispatched()); + assert!(!error.is_request()); + assert!(!report.contains("secret"), "{report}"); + assert!(!report.contains("document"), "{report}"); } #[tokio::test] diff --git a/litellm-rust/crates/http/src/request.rs b/litellm-rust/crates/http/src/request.rs index 5a3dd19ade2..11e19f7fa24 100644 --- a/litellm-rust/crates/http/src/request.rs +++ b/litellm-rust/crates/http/src/request.rs @@ -9,9 +9,7 @@ pub struct HeaderError { use litellm_core_utils::core_helpers::json_type_name; use serde_json::{Map, Value}; -/// Max characters of an upstream error body echoed across the call boundary -/// before truncation, so provider bodies are bounded and data-minimized. -const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256; +use crate::Error; pub enum HeaderPolicy<'a> { All, @@ -40,17 +38,15 @@ pub fn with_headers( }) } -pub async fn http_request( - request: reqwest::RequestBuilder, -) -> Result { - request.send().await +pub async fn http_request(request: reqwest::RequestBuilder) -> Result { + Ok(request.send().await?) } pub async fn execute_http_request( client: &reqwest::Client, request: reqwest::Request, -) -> Result { - client.execute(request).await +) -> Result { + Ok(client.execute(request).await?) } pub fn header_pairs(headers: &reqwest::header::HeaderMap) -> Vec<(String, String)> { @@ -65,14 +61,6 @@ pub fn header_pairs(headers: &reqwest::header::HeaderMap) -> Vec<(String, String .collect() } -pub fn truncate_error_body(body: &str) -> String { - if body.chars().count() <= UPSTREAM_ERROR_BODY_MAX_CHARS { - return body.to_string(); - } - let truncated: String = body.chars().take(UPSTREAM_ERROR_BODY_MAX_CHARS).collect(); - format!("{truncated}... (truncated)") -} - pub fn string_headers( context: &'static str, extra_headers: Option>, @@ -215,22 +203,6 @@ mod tests { assert_ne!(request.headers()["content-length"], "0"); } - #[test] - fn truncate_leaves_short_bodies_untouched() { - assert_eq!(truncate_error_body("short"), "short"); - } - - #[test] - fn truncate_bounds_long_bodies_by_characters() { - let body = "\u{00e9}".repeat(UPSTREAM_ERROR_BODY_MAX_CHARS + 10); - let truncated = truncate_error_body(&body); - assert!(truncated.ends_with("... (truncated)")); - assert_eq!( - truncated.chars().count(), - UPSTREAM_ERROR_BODY_MAX_CHARS + "... (truncated)".chars().count() - ); - } - #[test] fn string_headers_rejects_non_string_values() { let headers = Map::from_iter([("x-trace".to_string(), json!(7))]); diff --git a/litellm-rust/crates/http/src/tls.rs b/litellm-rust/crates/http/src/tls.rs index 01559b0d6ba..97319715ea0 100644 --- a/litellm-rust/crates/http/src/tls.rs +++ b/litellm-rust/crates/http/src/tls.rs @@ -256,21 +256,21 @@ fn certificates(path: &Path, source: TlsSource) -> Result Result, Error> { - std::fs::read(path).map_err(|cause| { + std::fs::read(path).map_err(|error| { Repr::Read { path: path.to_path_buf(), tls_source: source, - cause, + source: error, } .into() }) } -fn invalid_pem(path: &Path, source: TlsSource, cause: impl Into) -> Error { +fn invalid_pem(path: &Path, tls_source: TlsSource, fault: impl Into) -> Error { Repr::InvalidPem { path: path.to_path_buf(), - tls_source: source, - cause: cause.into(), + tls_source, + source: fault.into(), } .into() } diff --git a/litellm-rust/crates/http/src/websocket.rs b/litellm-rust/crates/http/src/websocket.rs index cfb994b6e9f..5998e1c5b9f 100644 --- a/litellm-rust/crates/http/src/websocket.rs +++ b/litellm-rust/crates/http/src/websocket.rs @@ -33,7 +33,7 @@ pub async fn connect( match timeout { Some(timeout) => tokio::time::timeout(timeout, handshake(client, url, headers)) .await - .map_err(|_| Repr::HandshakeTimeout)?, + .map_err(Repr::HandshakeTimeout)?, None => handshake(client, url, headers).await, } } @@ -117,7 +117,7 @@ fn http_url(url: &str) -> Result { other => return Err(Repr::WebSocketScheme(other.to_owned()).into()), }; url.set_scheme(scheme) - .map_err(|()| Repr::WebSocketScheme(url.scheme().to_owned()))?; + .map_err(|()| Repr::WebSocketSchemeRewrite(scheme))?; Ok(url) } diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index 689310e9765..3980d054900 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -3,10 +3,7 @@ use std::time::Duration; use litellm_auth::AuthServices; use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest}; -use litellm_http::{ - Client, - request::{truncate_error_body, with_default_headers}, -}; +use litellm_http::{Client, request::with_default_headers}; use litellm_llms::base_llm::{ auth::{Authenticated, ValidatedEnvironment, resolve_auth}, chat::transformation::BaseConfig, @@ -103,19 +100,13 @@ pub(super) async fn execute( ), )?; - let response = litellm_inference::outbound::send(outbound, http) - .await - .map_err(litellm_http::Error::from)?; + let response = litellm_inference::outbound::send(outbound, http).await?; let status = response.status(); let text = response.text().await.map_err(litellm_http::Error::from)?; if !status.is_success() { - return Err(litellm_http::Error::upstream( - status.as_u16(), - truncate_error_body(&text), - ) - .into()); + return Err(litellm_http::Error::upstream(status.as_u16(), text).into()); } let raw = RawResponse { body: text.clone() }; if let Some(observers) = observers { diff --git a/litellm-rust/crates/inference-messages/src/common_utils.rs b/litellm-rust/crates/inference-messages/src/common_utils.rs index 7394d7ae21e..b765b24304b 100644 --- a/litellm-rust/crates/inference-messages/src/common_utils.rs +++ b/litellm-rust/crates/inference-messages/src/common_utils.rs @@ -1,6 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::LlmProviders; use litellm_http::request::string_headers as shared_string_headers; -pub(super) use litellm_http::request::truncate_error_body; use litellm_llms::tencent::messages::transformation::TENCENT_MESSAGES_CONFIG; use litellm_llms::{ anthropic::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, @@ -77,7 +76,7 @@ mod tests { use rstest::rstest; - use super::{messages_provider, string_headers, truncate_error_body}; + use super::{messages_provider, string_headers}; use crate::Error; use litellm_core_utils::get_llm_provider_logic::LlmProviders; @@ -106,19 +105,6 @@ mod tests { assert!(messages_provider(provider, model).is_none()); } - #[test] - fn truncate_error_body_caps_long_payloads() { - let body = "x".repeat(400); - let truncated = truncate_error_body(&body); - assert!(truncated.ends_with("... (truncated)")); - let prefix_chars = truncated - .strip_suffix("... (truncated)") - .expect("truncated marker present") - .chars() - .count(); - assert_eq!(prefix_chars, 256); - } - #[test] fn string_headers_rejects_non_string_values() { let headers = json!({"x-count": 3}).as_object().unwrap().clone(); diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 5a067db937d..5b7fd2d45b9 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -26,7 +26,7 @@ use serde_json::Value; use super::{ Error, MessagesCall, MessagesCallResponse, MessagesRoute, - common_utils::{MessagesProvider, string_headers, truncate_error_body}, + common_utils::{MessagesProvider, string_headers}, constants::MESSAGES_TIMEOUT_SECS, route::Messages, types::invalid_request, @@ -292,9 +292,7 @@ async fn send( body, Some(timeout.unwrap_or(Duration::from_secs(MESSAGES_TIMEOUT_SECS))), )?; - litellm_inference::outbound::send(request, http) - .await - .map_err(network) + Ok(litellm_inference::outbound::send(request, http).await?) } async fn provider_error(response: reqwest::Response) -> Error { @@ -302,7 +300,7 @@ async fn provider_error(response: reqwest::Response) -> Error { match response.text().await { Ok(text) => { log_error_body(status, &text); - litellm_http::Error::upstream(status, truncate_error_body(&text)).into() + litellm_http::Error::upstream(status, text).into() } Err(error) => network(error), } diff --git a/litellm-rust/crates/inference-ocr/Cargo.toml b/litellm-rust/crates/inference-ocr/Cargo.toml index 21ec013f873..8d18b1f3aa6 100644 --- a/litellm-rust/crates/inference-ocr/Cargo.toml +++ b/litellm-rust/crates/inference-ocr/Cargo.toml @@ -25,6 +25,7 @@ tokio.workspace = true tracing.workspace = true [dev-dependencies] +litellm-error.workspace = true litellm-auth-aws.workspace = true litellm-auth-gcp.workspace = true litellm-host-native.workspace = true diff --git a/litellm-rust/crates/inference-ocr/src/route.rs b/litellm-rust/crates/inference-ocr/src/route.rs index 85125452bda..74ff829e8ad 100644 --- a/litellm-rust/crates/inference-ocr/src/route.rs +++ b/litellm-rust/crates/inference-ocr/src/route.rs @@ -43,7 +43,10 @@ fn caller_token_provider(services: HostServices) -> TokenProviderHandle { .call(OcrOp::AcquireAzureAdToken) .await .map_err(|error| { - litellm_auth::Error::CredentialAcquisition(error.to_string().into()) + litellm_auth::Error::CredentialAcquisition(litellm_auth::ErrorDetail::failed( + "host Azure AD token request", + error, + )) }) } }) diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs b/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs index 9bff7490d58..7370507017a 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/azure_ai.rs @@ -232,7 +232,7 @@ async fn credential_precedence( true, json!({"azure_ad_token": "oidc/assertion", "client_id": "client", "tenant_id": "tenant"}), numbered_token, - |error: &Error| matches!(error, Error::Auth(litellm_auth::Error::InvalidConfiguration(detail)) if detail.to_string() == "unsupported OIDC reference"), + |error: &Error| litellm_error::Report::new(error).to_string().contains("unsupported OIDC reference"), 0 )] #[case::empty_provider_token_ignores_static_token( diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/main.rs b/litellm-rust/crates/inference-ocr/tests/ocr/main.rs index 5a905f907d2..7573cf17e55 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/main.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/main.rs @@ -223,11 +223,11 @@ impl LocalOcrHost { impl litellm_host_native::services::HostCallHandler for LocalOcrHost { async fn handle_host_call(&self, op: OcrOp) -> Result<(), Error> { match op { - OcrOp::AcquireAzureAdToken(_) => { - Err(Error::Auth(litellm_auth::Error::CredentialAcquisition( - "OCR host has no Azure AD token provider".into(), - ))) - } + OcrOp::AcquireAzureAdToken(_) => Err(Error::Auth( + litellm_auth::Error::CredentialAcquisition(litellm_auth::ErrorDetail::Unresolved { + subject: "OCR host Azure AD token provider", + }), + )), } } } diff --git a/litellm-rust/crates/inference-ocr/tests/ocr/vertex_ai.rs b/litellm-rust/crates/inference-ocr/tests/ocr/vertex_ai.rs index 81b52d076a5..dba12de60f6 100644 --- a/litellm-rust/crates/inference-ocr/tests/ocr/vertex_ai.rs +++ b/litellm-rust/crates/inference-ocr/tests/ocr/vertex_ai.rs @@ -71,7 +71,8 @@ async fn invalid_credentials_fail_before_sending() { .await .unwrap_err(); - assert!(error.to_string().contains("vertex_credentials"), "{error}"); + let report = litellm_error::Report::new(&error).to_string(); + assert!(report.contains("vertex_credentials"), "{report}"); } #[rstest] @@ -91,11 +92,10 @@ async fn a_request_controlled_api_base_is_rejected_before_vertex_auth( let error = perform(request).await.unwrap_err(); + let report = litellm_error::Report::new(&error).to_string(); assert!( - error - .to_string() - .contains("request-controlled Vertex AI endpoint"), - "{error}" + report.contains("request-controlled Vertex AI endpoint"), + "{report}" ); } diff --git a/litellm-rust/crates/inference-responses/src/handler.rs b/litellm-rust/crates/inference-responses/src/handler.rs index b94638c4c04..c370a5e7645 100644 --- a/litellm-rust/crates/inference-responses/src/handler.rs +++ b/litellm-rust/crates/inference-responses/src/handler.rs @@ -83,17 +83,11 @@ pub(super) async fn execute( &wire.body, Some(call.timeout.unwrap_or(Duration::from_secs(600))), )?; - let response = litellm_inference::outbound::send(outbound, &route.http) - .await - .map_err(network)?; + let response = litellm_inference::outbound::send(outbound, &route.http).await?; let status = response.status().as_u16(); if !response.status().is_success() { let body = response.text().await.map_err(network)?; - return Err(litellm_http::Error::upstream( - status, - litellm_http::request::truncate_error_body(&body), - ) - .into()); + return Err(litellm_http::Error::upstream(status, body).into()); } if stream { let headers = response diff --git a/litellm-rust/crates/inference-transcription/src/handler.rs b/litellm-rust/crates/inference-transcription/src/handler.rs index 7ad1f63b4fb..93be7387121 100644 --- a/litellm-rust/crates/inference-transcription/src/handler.rs +++ b/litellm-rust/crates/inference-transcription/src/handler.rs @@ -2,7 +2,7 @@ use std::time::Duration; use litellm_http::{ Client, - request::{string_headers, truncate_error_body, with_default_headers}, + request::{string_headers, with_default_headers}, }; use litellm_llms::base_llm::{ audio_transcription::transformation::BaseAudioTranscriptionConfig, @@ -55,15 +55,11 @@ pub(super) async fn execute( .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)), ), )?; - let response = litellm_inference::outbound::send(outbound, http) - .await - .map_err(litellm_http::Error::from)?; + let response = litellm_inference::outbound::send(outbound, http).await?; let status = response.status(); let text = response.text().await.map_err(litellm_http::Error::from)?; if !status.is_success() { - return Err( - litellm_http::Error::upstream(status.as_u16(), truncate_error_body(&text)).into(), - ); + return Err(litellm_http::Error::upstream(status.as_u16(), text).into()); } let response_json = serde_json::from_str(&text).map_err(|error| { Error::InvalidResponse(litellm_llms::ErrorDetail::invalid( diff --git a/litellm-rust/crates/inference/src/outbound.rs b/litellm-rust/crates/inference/src/outbound.rs index 9d9b1e606c2..b0cc71d425d 100644 --- a/litellm-rust/crates/inference/src/outbound.rs +++ b/litellm-rust/crates/inference/src/outbound.rs @@ -13,10 +13,10 @@ use serde_json::Value; pub async fn send( request: reqwest::Request, client: &litellm_http::Client, -) -> Result { - client.execute(request).await.inspect(|response| { - tracing::Span::current().record("status", response.status().as_u16()); - }) +) -> Result { + let response = client.execute(request).await?; + tracing::Span::current().record("status", response.status().as_u16()); + Ok(response) } /// Header credentials are already in `headers`; SigV4 is applied here, over the diff --git a/litellm-rust/crates/inference/tests/caching.rs b/litellm-rust/crates/inference/tests/caching.rs index a0a6a757565..02c2edd95dd 100644 --- a/litellm-rust/crates/inference/tests/caching.rs +++ b/litellm-rust/crates/inference/tests/caching.rs @@ -417,7 +417,7 @@ impl litellm_cache::BaseCache for UnavailableCache { _: &str, _: &Self::Context, ) -> Result, litellm_cache::Error> { - Err(litellm_cache::Error::Unavailable) + Err(litellm_cache::Error::Unavailable(None)) } fn set_cache( @@ -426,7 +426,7 @@ impl litellm_cache::BaseCache for UnavailableCache { _: Self::Value, _: &Self::Context, ) -> Result<(), litellm_cache::Error> { - Err(litellm_cache::Error::Unavailable) + Err(litellm_cache::Error::Unavailable(None)) } } diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 2d3ea85d1fc..b27b9dc12dc 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -37,6 +37,7 @@ tokio = { workspace = true, features = ["sync"] } url.workspace = true [dev-dependencies] +litellm-error.workspace = true litellm-http = { workspace = true, features = ["test-support"] } aws-smithy-eventstream = "=0.61.4" aws-smithy-types = "1.6.1" diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs index 8cbf2455e01..7da685febf9 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs @@ -26,7 +26,7 @@ pub(super) async fn resolve_entra( .get_azure_ad_token(config, env_lookup) .await .or_else(|error| match error { - litellm_auth::Error::EmptyCallerCredential(_) => Ok(None), + litellm_auth::Error::EmptyCallerCredential { .. } => Ok(None), other => Err(other), }) .map(|credential| { @@ -48,7 +48,7 @@ pub(super) fn validate_destination( && credential_source != InputSource::Request { return Err(litellm_auth::Error::InvalidConfiguration( - "host credentials cannot be sent to a request-controlled Azure endpoint".into(), + litellm_auth::ErrorDetail::RequestControlledEndpoint { provider: "Azure" }, ) .into()); } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 48de38c44c5..237ecb89962 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -185,7 +185,10 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { ) -> Result { let endpoint = nonblank(request.connection.api_base.clone()) .or_else(|| nonblank(request.connection.secret(AZURE_DI_ENDPOINT_ENV))) - .ok_or_else(|| Error::Auth(litellm_auth::Error::ProviderAuthentication("Missing Azure Document Intelligence API Base - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT or pass api_base".into())))?; + .ok_or(Error::Auth(litellm_auth::Error::MissingApiBase { + provider: "Azure Document Intelligence", + guidance: "Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT or pass api_base", + }))?; self.build_ocr_url( &endpoint, &request.model, @@ -842,7 +845,7 @@ mod tests { .unwrap_err(); assert!( - error + litellm_error::Report::new(&error) .to_string() .contains("request-controlled Azure endpoint") ); diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs index 83d88bbb3cb..03d89f699bc 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs @@ -302,7 +302,7 @@ mod tests { .unwrap_err(); assert!( - error + litellm_error::Report::new(&error) .to_string() .contains("request-controlled Azure endpoint") ); diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs index c8ddad46898..bc52cfa156a 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -158,8 +158,8 @@ impl Error { | Self::InvalidProvider(_) | Self::InvalidModel { .. } | Self::Params(_) - | Self::Headers(_) - | Self::Auth(litellm_auth::Error::MissingParam { .. }) => true, + | Self::Headers(_) => true, + Self::Auth(error) => error.is_missing_setting(), Self::Http(error) => error.is_request(), _ => false, } diff --git a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs index 72958fd6cb9..79c30df9b19 100644 --- a/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs @@ -1,6 +1,6 @@ use std::sync::LazyLock; -use litellm_auth::{AwsParams, CredentialPlacement, ParamSpec, SecretValue}; +use litellm_auth::{AwsParams, CredentialPlacement, ParamKind, ParamSpec, SecretValue}; use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region}; use litellm_core_utils::settings::resolve_non_empty; use litellm_http::request::{header_value, with_header, without_headers}; @@ -36,6 +36,7 @@ const WORKSPACE_FIELDS: &[&str] = &[ const WORKSPACE_HEADER: &str = "anthropic-workspace-id"; static WORKSPACE: ParamSpec = ParamSpec { setting: "workspace_id", + kind: ParamKind::Setting, wire: WORKSPACE_FIELDS, module_global: None, env: WORKSPACE_ENVS, diff --git a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs index dadab413fe4..95e142589f8 100644 --- a/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs @@ -195,11 +195,10 @@ impl CohereParseConfig { .and_then(env_lookup) .filter(|key| !key.trim().is_empty()) }) - .ok_or_else(|| { - Error::Auth(litellm_auth::Error::ProviderAuthentication( - "Missing COHERE_API_KEY - set it in the environment or pass api_key".into(), - )) - })?; + .ok_or(Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Cohere", + environment_variable: COHERE_API_KEY_ENV, + }))?; Ok( std::iter::once(("Authorization".into(), format!("Bearer {key}"))) .chain(connection.extra_headers.clone()) diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs index 810462c57f8..d760da9073e 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs @@ -16,7 +16,9 @@ pub(super) fn vertex_config(request: &PreparedOcrRequest) -> Result Result<(), Error> { if connection.api_base.is_some() && connection.api_base_source == InputSource::Request { return Err(litellm_auth::Error::InvalidConfiguration( - "credentials cannot be sent to a request-controlled Vertex AI endpoint".into(), + litellm_auth::ErrorDetail::RequestControlledEndpoint { + provider: "Vertex AI", + }, ) .into()); } diff --git a/litellm-rust/crates/model-catalog/Cargo.toml b/litellm-rust/crates/model-catalog/Cargo.toml index 570e68ca4fd..2c2866d7090 100644 --- a/litellm-rust/crates/model-catalog/Cargo.toml +++ b/litellm-rust/crates/model-catalog/Cargo.toml @@ -20,4 +20,5 @@ time.workspace = true [dev-dependencies] jsonschema = { version = "0.55.1", default-features = false } +litellm-error.workspace = true rstest.workspace = true diff --git a/litellm-rust/crates/model-catalog/src/catalog.rs b/litellm-rust/crates/model-catalog/src/catalog.rs index 0b113436a5b..37412ef1e14 100644 --- a/litellm-rust/crates/model-catalog/src/catalog.rs +++ b/litellm-rust/crates/model-catalog/src/catalog.rs @@ -20,11 +20,15 @@ pub struct IntegrityLimits { pub min_reference_ratio: f64, } -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum AliasIssue { + #[error("model {model:?} has aliases that are not an array")] InvalidList { model: String }, + #[error("model {model:?} has an alias that is not a string")] InvalidName { model: String }, + #[error("alias {alias:?} of model {model:?} is already a model name")] CanonicalCollision { model: String, alias: String }, + #[error("alias {alias:?} of model {model:?} is already an alias of another model")] AliasCollision { model: String, alias: String }, } diff --git a/litellm-rust/crates/model-catalog/src/error.rs b/litellm-rust/crates/model-catalog/src/error.rs index edb6ba1eb11..00902858be4 100644 --- a/litellm-rust/crates/model-catalog/src/error.rs +++ b/litellm-rust/crates/model-catalog/src/error.rs @@ -2,26 +2,20 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum Error { - /// The body is not valid JSON, or a model entry fails typed deserialization. - #[error("invalid JSON: {0}")] + #[error("invalid catalog JSON")] Json(#[from] serde_json::Error), - /// The catalog has no entries at all. #[error("catalog is empty")] Empty, - /// A non-reserved top level value is not a JSON object. #[error("model {model:?} must be an object")] EntryNotObject { model: String }, - /// Canonical entry count is under the configured minimum. #[error("catalog has {actual} models, below minimum {minimum}")] BelowMinimum { actual: usize, minimum: usize }, - /// Canonical entry count is under the configured reference ratio. #[error("catalog has {actual} models, below {ratio} of reference count {reference}")] Shrunk { actual: usize, reference: usize, ratio: f64, }, - /// The configured minimum reference ratio is not finite or outside `[0, 1]`. #[error("minimum reference ratio must be finite and between zero and one")] InvalidRatio, } diff --git a/litellm-rust/crates/model-catalog/src/validation.rs b/litellm-rust/crates/model-catalog/src/validation.rs index 73f08a865c1..6dd4174148c 100644 --- a/litellm-rust/crates/model-catalog/src/validation.rs +++ b/litellm-rust/crates/model-catalog/src/validation.rs @@ -5,24 +5,57 @@ use thiserror::Error; use crate::{AliasIssue, Catalog, ModelInfo, UtcHours, Weekday}; -/// A registry entry violates the checked-in catalog contract. #[derive(Debug, Error)] pub enum RegistryValidationError { - #[error("{reason}")] - Entry { model: String, reason: String }, - #[error("alias issue: {0:?}")] + #[error("invalid registry entry {model:?}")] + Entry { + model: String, + #[source] + reason: EntryViolation, + }, + #[error(transparent)] Alias(AliasIssue), } -/// Validate one registry entry without restricting the tolerant catalog reader. +#[derive(Debug, Error)] +pub enum EntryViolation { + #[error("entry must be an object")] + NotObject, + #[error("aliases must be an array")] + AliasesNotArray, + #[error("aliases must contain strings")] + AliasNotString, + #[error("entry does not match the model schema")] + Schema(#[source] serde_json::Error), + #[error("litellm_provider is required")] + MissingProvider, + #[error("invalid deprecation_date")] + DeprecationDate(#[source] time::error::Parse), + #[error("off_peak_pricing needs hours or windows")] + OffPeakWithoutHours, + #[error("off_peak_pricing.hours_utc is invalid")] + InvalidHours, + #[error("off_peak_pricing.windows is empty")] + EmptyWindows, + #[error("off_peak_pricing.weekdays is invalid")] + InvalidWeekdays, + #[error("typed entry could not be serialized")] + Serialize(#[source] serde_json::Error), + #[error("entry has an unknown field, null, or changed value: {}", fields.join(", "))] + ChangedFields { fields: Vec }, + #[error("{field} must be nonnegative")] + Negative { field: String }, + #[error("{field} must be at least one")] + UpliftBelowOne { field: String }, +} + pub fn validate_model_entry(model: &str, value: &Value) -> Result<(), RegistryValidationError> { - validate_entry_inner(model, value).map_err(|reason| RegistryValidationError::Entry { + validate_entry_inner(value).map_err(|reason| RegistryValidationError::Entry { model: model.to_owned(), reason, }) } -/// Check every model and alias in a parsed catalog against registry rules. pub fn validate_registry(catalog: &Catalog) -> Result<(), RegistryValidationError> { if let Some(issue) = catalog.alias_issues().first() { return Err(RegistryValidationError::Alias(issue.clone())); @@ -53,70 +86,61 @@ fn keys(value: &Map) -> BTreeSet { value.keys().cloned().collect() } -fn symmetric_difference(left: &BTreeSet, right: &BTreeSet) -> BTreeSet { +fn symmetric_difference(left: &BTreeSet, right: &BTreeSet) -> Vec { left.symmetric_difference(right).cloned().collect() } -fn validate_entry_inner(model_name: &str, value: &Value) -> Result<(), String> { - let object = value - .as_object() - .ok_or_else(|| format!("{model_name} must be an object"))?; +fn validate_entry_inner(value: &Value) -> Result<(), EntryViolation> { + let object = value.as_object().ok_or(EntryViolation::NotObject)?; if let Some(aliases) = object.get("aliases") { - let names = aliases - .as_array() - .ok_or_else(|| format!("{model_name}.aliases must be an array"))?; + let names = aliases.as_array().ok_or(EntryViolation::AliasesNotArray)?; if names.iter().any(|name| !name.is_string()) { - return Err(format!("{model_name}.aliases must contain strings")); + return Err(EntryViolation::AliasNotString); } } - let info: ModelInfo = - serde_json::from_value(value.clone()).map_err(|error| format!("{model_name}: {error}"))?; + let info: ModelInfo = serde_json::from_value(value.clone()).map_err(EntryViolation::Schema)?; if info.litellm_provider.is_none() { - return Err(format!("{model_name}.litellm_provider is required")); + return Err(EntryViolation::MissingProvider); } - validate_dates_and_windows(model_name, &info)?; - let serialized = serde_json::to_value(info).map_err(|error| error.to_string())?; + validate_dates_and_windows(&info)?; + let serialized = serde_json::to_value(info).map_err(EntryViolation::Serialize)?; let mut expected = object.clone(); expected.remove("aliases"); if !json_eq(&Value::Object(expected.clone()), &serialized) { let actual = serialized .as_object() .expect("ModelInfo serializes as an object"); - return Err(format!( - "{model_name} has an unknown field, null, or changed value: {:?}", - symmetric_difference(&keys(&expected), &keys(actual)) - )); + return Err(EntryViolation::ChangedFields { + fields: symmetric_difference(&keys(&expected), &keys(actual)), + }); } - check_prices(model_name, value) + check_prices("", value) } -fn validate_dates_and_windows(model_name: &str, info: &ModelInfo) -> Result<(), String> { +fn validate_dates_and_windows(info: &ModelInfo) -> Result<(), EntryViolation> { if let Some(date) = &info.deprecation_date { let format = time::format_description::parse_borrowed::<2>("[year]-[month]-[day]").unwrap(); - time::Date::parse(date, &format) - .map_err(|error| format!("{model_name}.deprecation_date: {error}"))?; + time::Date::parse(date, &format).map_err(EntryViolation::DeprecationDate)?; } let Some(pricing) = &info.off_peak_pricing else { return Ok(()); }; if pricing.hours_utc.is_none() && pricing.windows.is_none() { - return Err(format!( - "{model_name}.off_peak_pricing needs hours or windows" - )); + return Err(EntryViolation::OffPeakWithoutHours); } if let Some(hours) = &pricing.hours_utc { validate_hours(hours)?; } if let Some(windows) = &pricing.windows { if windows.is_empty() { - return Err(format!("{model_name}.off_peak_pricing.windows is empty")); + return Err(EntryViolation::EmptyWindows); } windows.iter().try_for_each(|window| { validate_hours(&window.hours_utc)?; if let Some(days) = &window.weekdays && (days.is_empty() || days.iter().any(|day| !valid_weekday(day))) { - return Err(format!("{model_name}.off_peak_pricing.weekdays is invalid")); + return Err(EntryViolation::InvalidWeekdays); } Ok(()) })?; @@ -124,13 +148,13 @@ fn validate_dates_and_windows(model_name: &str, info: &ModelInfo) -> Result<(), Ok(()) } -fn validate_hours(hours: &UtcHours) -> Result<(), String> { +fn validate_hours(hours: &UtcHours) -> Result<(), EntryViolation> { let values = match hours { UtcHours::Single(value) => std::slice::from_ref(value), UtcHours::Multiple(values) => values.as_slice(), }; if values.is_empty() || values.iter().any(|value| !valid_utc_window(value)) { - return Err("off_peak_pricing.hours_utc is invalid".into()); + return Err(EntryViolation::InvalidHours); } Ok(()) } @@ -176,7 +200,7 @@ fn valid_weekday(day: &Weekday) -> bool { } } -fn check_prices(path: &str, value: &Value) -> Result<(), String> { +fn check_prices(path: &str, value: &Value) -> Result<(), EntryViolation> { let Some(object) = value.as_object() else { return Ok(()); }; @@ -188,13 +212,17 @@ fn check_prices(path: &str, value: &Value) -> Result<(), String> { && let Some(number) = field.as_f64() && number < 0.0 { - return Err(format!("{field_path} must be nonnegative")); + return Err(EntryViolation::Negative { + field: relative(&field_path), + }); } if key.contains("uplift_multiplier") && let Some(number) = field.as_f64() && number < 1.0 { - return Err(format!("{field_path} must be at least one")); + return Err(EntryViolation::UpliftBelowOne { + field: relative(&field_path), + }); } if matches!(key.as_str(), "range" | "max_results_range") && field.as_array().is_some_and(|values| { @@ -203,7 +231,9 @@ fn check_prices(path: &str, value: &Value) -> Result<(), String> { .any(|value| value.as_f64().is_some_and(|n| n < 0.0)) }) { - return Err(format!("{field_path} must be nonnegative")); + return Err(EntryViolation::Negative { + field: relative(&field_path), + }); } if matches!(key.as_str(), "metadata" | "provider_specific_entry") { return Ok(()); @@ -216,3 +246,7 @@ fn check_prices(path: &str, value: &Value) -> Result<(), String> { } }) } + +fn relative(field_path: &str) -> String { + field_path.trim_start_matches('.').to_owned() +} diff --git a/litellm-rust/crates/model-catalog/tests/registry_validation.rs b/litellm-rust/crates/model-catalog/tests/registry_validation.rs index 67589010aba..3b65ef13d78 100644 --- a/litellm-rust/crates/model-catalog/tests/registry_validation.rs +++ b/litellm-rust/crates/model-catalog/tests/registry_validation.rs @@ -52,9 +52,9 @@ fn fallback_generalizations_are_typed(repo_root: PathBuf) { #[case::null_aliases(serde_json::json!({"litellm_provider": "test", "aliases": null}), "aliases must be an array")] #[ignore] fn registry_validation_rejects_malformed_entries(#[case] entry: Value, #[case] expected: &str) { + let error = validate_model_entry("test", &entry).unwrap_err(); assert!( - validate_model_entry("test", &entry) - .unwrap_err() + litellm_error::Report::new(&error) .to_string() .contains(expected) ); diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index ad86d1f5e79..b4ef36f53c6 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -27,6 +27,7 @@ litellm-host.workspace = true bytes.workspace = true futures-util.workspace = true litellm-cache.workspace = true +litellm-error.workspace = true litellm-cache-memory.workspace = true litellm-cache-redis.workspace = true litellm-cache-response.workspace = true diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index e190b12e665..9dd0d9eb3a1 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -13,9 +13,52 @@ use pyo3::{ }; fn cache_error(error: Error) -> PyErr { + let message = litellm_error::Report::new(&error).to_string(); match error { - Error::InvalidEntry => PyValueError::new_err(error.to_string()), - Error::UnsupportedOperation => PyNotImplementedError::new_err(error.to_string()), - _ => PyRuntimeError::new_err(error.to_string()), + Error::InvalidEntry(_) => PyValueError::new_err(message), + Error::UnsupportedOperation => PyNotImplementedError::new_err(message), + Error::Unavailable(_) | Error::UnscopedFlush | Error::MissingPrompt => { + PyRuntimeError::new_err(message) + } + } +} + +#[cfg(test)] +mod tests { + use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + }; + use rstest::rstest; + + use super::{Error, cache_error}; + + fn refused() -> std::io::Error { + std::io::Error::other("connection refused") + } + + #[rstest] + #[case::unavailable( + Error::unavailable(refused()), + false, + "cache is unavailable: connection refused" + )] + #[case::invalid_entry( + Error::invalid_entry(refused()), + true, + "invalid cache entry: connection refused" + )] + fn python_sees_the_class_and_the_whole_chain( + #[case] error: Error, + #[case] caller_fault: bool, + #[case] message: &str, + ) { + Python::initialize(); + Python::attach(|py| { + let failure = cache_error(error); + assert_eq!(failure.is_instance_of::(py), caller_fault); + assert_eq!(failure.is_instance_of::(py), !caller_fault); + assert_eq!(failure.value(py).to_string(), message); + }); } } diff --git a/litellm-rust/crates/python-bridge/src/cache/python/host.rs b/litellm-rust/crates/python-bridge/src/cache/python/host.rs index d8f8ac7c181..5d7ec9134b7 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/host.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/host.rs @@ -105,19 +105,19 @@ impl PythonCache { } match self.pending.take() { Some(Pending::Lookup(reply)) => { - let value = result.map_err(|_| Error::Unavailable).and_then(|value| { + let value = result.map_err(Error::unavailable).and_then(|value| { if value.bind(py).is_none() { Ok(None) } else { from_py(value.bind(py)) .map(Some) - .map_err(|_| Error::InvalidEntry) + .map_err(Error::invalid_entry) } }); reply.send(value); } Some(Pending::Store(reply)) => { - reply.send(result.map(|_| ()).map_err(|_| Error::Unavailable)); + reply.send(result.map(|_| ()).map_err(Error::unavailable)); } None => { return Err(pyo3::exceptions::PyRuntimeError::new_err( diff --git a/litellm-rust/crates/python-bridge/src/cache/python/service.rs b/litellm-rust/crates/python-bridge/src/cache/python/service.rs index f215e5e97f5..38872b85d30 100644 --- a/litellm-rust/crates/python-bridge/src/cache/python/service.rs +++ b/litellm-rust/crates/python-bridge/src/cache/python/service.rs @@ -17,18 +17,21 @@ fn from_python(value: Value) -> Result { let output = match value.get(STREAM_EVENTS_KEY) { Some(events) => { let events: Vec = - serde_json::from_value(events.clone()).map_err(|_| Error::InvalidEntry)?; + serde_json::from_value(events.clone()).map_err(Error::invalid_entry)?; CachedOutput::Stream(events.concat()) } None => CachedOutput::Response(value), }; - serde_json::to_value(ResponseEnvelope::new("messages", output)).map_err(|_| Error::InvalidEntry) + serde_json::to_value(ResponseEnvelope::new("messages", output)).map_err(Error::invalid_entry) } fn to_python(value: Value) -> Result { let envelope: ResponseEnvelope> = - serde_json::from_value(value).map_err(|_| Error::InvalidEntry)?; - match envelope.decode("messages").ok_or(Error::InvalidEntry)? { + serde_json::from_value(value).map_err(Error::invalid_entry)?; + match envelope + .decode("messages") + .ok_or(Error::InvalidEntry(None))? + { CachedOutput::Response(response) => Ok(response), CachedOutput::Stream(text) => Ok(serde_json::json!({ STREAM_EVENTS_KEY: text.split_inclusive("\n\n").collect::>() @@ -56,7 +59,7 @@ pub(in crate::cache) fn service>( namespace: String, ) -> std::sync::Arc where - P::Error: From, + P::Error: From + std::error::Error, { std::sync::Arc::new(PythonCacheService { services, @@ -69,7 +72,7 @@ where impl> ResponseCacheService for PythonCacheService

where - P::Error: From, + P::Error: From + std::error::Error, { fn config(&self) -> &ResponseCacheConfig { &self.config @@ -84,7 +87,7 @@ where self.services .call(|reply| CacheCall::Lookup { reply }) .await - .map_err(|_| Error::Unavailable)?? + .map_err(Error::unavailable)?? .map(from_python) .transpose() }) @@ -101,7 +104,7 @@ where self.services .call(|reply| CacheCall::Store { value, reply }) .await - .map_err(|_| Error::Unavailable)? + .map_err(Error::unavailable)? }) } } diff --git a/litellm-rust/crates/python-bridge/src/cache/selection.rs b/litellm-rust/crates/python-bridge/src/cache/selection.rs index cd19533fc73..00fc7cd822f 100644 --- a/litellm-rust/crates/python-bridge/src/cache/selection.rs +++ b/litellm-rust/crates/python-bridge/src/cache/selection.rs @@ -39,7 +39,7 @@ impl Selection { services: HostServices

, ) -> (Option, CacheOptions) where - P::Error: From, + P::Error: From + std::error::Error, { let service = match self.backend { Backend::Disabled => None, diff --git a/litellm-rust/crates/python-bridge/src/routes/clickhouse_spend.rs b/litellm-rust/crates/python-bridge/src/routes/clickhouse_spend.rs index f8df7a9de23..80d7bd5360e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/clickhouse_spend.rs +++ b/litellm-rust/crates/python-bridge/src/routes/clickhouse_spend.rs @@ -8,24 +8,13 @@ use pyo3::{ }; fn map_error(error: Error) -> PyErr { - use litellm_storage_clickhouse::Error as StorageError; - match error { - Error::InsertTooLarge | Error::Storage(StorageError::InsertTooLarge) => { - PyOverflowError::new_err(error.to_string()) - } - Error::InvalidRow - | Error::InvalidLimit(_) - | Error::InvalidSchema - | Error::Storage( - StorageError::InvalidRow - | StorageError::InvalidLimit(_) - | StorageError::InvalidTable - | StorageError::InvalidSchema - | StorageError::EmptySql - | StorageError::InvalidParameters - | StorageError::InvalidQuery, - ) => PyValueError::new_err(error.to_string()), - Error::Migration(_) | Error::Storage(_) => PyRuntimeError::new_err(error.to_string()), + let message = litellm_error::Report::new(&error).to_string(); + if error.is_insert_too_large() { + PyOverflowError::new_err(message) + } else if error.is_request() { + PyValueError::new_err(message) + } else { + PyRuntimeError::new_err(message) } } @@ -99,26 +88,22 @@ mod tests { use super::*; use rstest::rstest; + use litellm_storage_clickhouse::Error as StorageError; + #[rstest] - #[case::row(Error::InvalidRow, "ValueError")] - #[case::insert_limit(Error::InvalidLimit("CLICKHOUSE_TRACE_MAX_INSERT_BYTES"), "ValueError")] - #[case::insert_timeout( - Error::Storage(litellm_storage_clickhouse::Error::InvalidLimit( - "CLICKHOUSE_INSERT_TIMEOUT_SECONDS" - )), + #[case::timestamp(Error::NonIntegerTimestamp { field: "start_time" }, "ValueError")] + #[case::insert_limit( + StorageError::InvalidLimit("CLICKHOUSE_TRACE_MAX_INSERT_BYTES").into(), "ValueError" )] - #[case::insert_budget(Error::InsertTooLarge, "OverflowError")] - #[case::schema( - Error::Storage(litellm_storage_clickhouse::Error::SchemaFailed(503)), - "RuntimeError" - )] + #[case::insert_budget(StorageError::InsertTooLarge.into(), "OverflowError")] + #[case::schema(StorageError::SchemaFailed(503).into(), "RuntimeError")] #[case::migration( Error::Migration(sqlx::migrate::MigrateError::VersionMismatch(6)), "RuntimeError" )] #[case::storage( - Error::Storage(litellm_storage_clickhouse::Error::InvalidUrl), + Config::new("spend".into(), "ftp://localhost", 1).err().unwrap(), "RuntimeError" )] fn spend_failures_preserve_public_exception_types( @@ -127,7 +112,7 @@ mod tests { ) { Python::initialize(); Python::attach(|py| { - let message = error.to_string(); + let message = litellm_error::Report::new(&error).to_string(); let exception = map_error(error); assert_eq!(exception.get_type(py).name().unwrap(), exception_name); assert_eq!( @@ -136,4 +121,15 @@ mod tests { ); }); } + + #[test] + fn spend_failures_render_their_cause() { + let error = Config::new("spend".into(), "not a url", 1).err().unwrap(); + let cause = std::error::Error::source(&error).unwrap().to_string(); + Python::initialize(); + Python::attach(|py| { + let message = map_error(error).value(py).str().unwrap().to_string(); + assert!(message.ends_with(&cause), "{message}"); + }); + } } diff --git a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs index 2c26311231f..4b10eed087f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/token_counter.rs +++ b/litellm-rust/crates/python-bridge/src/routes/token_counter.rs @@ -2,6 +2,7 @@ use crate::execution::run_async; use std::sync::Arc; use std::{num::NonZero, thread::available_parallelism}; +use litellm_error::{Report, Source}; use litellm_host_python::enter_native; use litellm_token_counter::{ CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, @@ -50,10 +51,10 @@ impl TokenCounter { let _slot = encode_slots .acquire_owned() .await - .map_err(|error| Error::Task(error.to_string()))?; + .map_err(|error| Error::Task(Source::new(error)))?; tokio::task::spawn_blocking(move || count_body(&counter, &body)) .await - .map_err(|error| Error::Task(error.to_string()))? + .map_err(|error| Error::Task(Source::new(error)))? }, token_count_error_to_pyerr, ) @@ -70,19 +71,12 @@ fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result PyErr { - let message = error.to_string(); - match error { - Error::Load(_) - | Error::Ranks(_) - | Error::UnicodeClasses - | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message), - Error::RequestParse(_) - | Error::MissingInput - | Error::FloatText - | Error::ContentBlock - | Error::ArrayItems - | Error::JsonSerialization(_) - | Error::JsonUtf8(_) => RustBridgeDeclined::new_err(message), - Error::Encode(_) | Error::Decode(_) | Error::Task(_) => PyRuntimeError::new_err(message), + let message = Report::new(&error).to_string(); + if error.is_unsupported() { + RustBridgeDeclined::new_err(message) + } else if error.is_load() { + PyValueError::new_err(message) + } else { + PyRuntimeError::new_err(message) } } diff --git a/litellm-rust/crates/python-bridge/src/secrets/error.rs b/litellm-rust/crates/python-bridge/src/secrets/error.rs index 5ccfd0a1554..17f3bdc4cad 100644 --- a/litellm-rust/crates/python-bridge/src/secrets/error.rs +++ b/litellm-rust/crates/python-bridge/src/secrets/error.rs @@ -14,11 +14,15 @@ impl fmt::Debug for PythonSecretError { } pub(super) fn external_error(py: Python<'_>, error: PyErr) -> Error { - Error::ExternalManager(Box::new(PythonSecretError(error.into_value(py)))) + Error::ExternalManager(litellm_error::Source::new(PythonSecretError( + error.into_value(py), + ))) } pub(super) fn read_error(py: Python<'_>, error: PyErr) -> Error { - Error::ExternalRead(Box::new(PythonSecretError(error.into_value(py)))) + Error::ExternalRead(litellm_error::Source::new(PythonSecretError( + error.into_value(py), + ))) } pub(crate) fn python_error(py: Python<'_>, error: &Error) -> Option { diff --git a/litellm-rust/crates/python-bridge/src/tokenizer.rs b/litellm-rust/crates/python-bridge/src/tokenizer.rs index df219d55eb6..1702b482cb3 100644 --- a/litellm-rust/crates/python-bridge/src/tokenizer.rs +++ b/litellm-rust/crates/python-bridge/src/tokenizer.rs @@ -8,6 +8,8 @@ use std::sync::Arc; #[cfg(feature = "fast")] use std::sync::OnceLock; +#[cfg(any(feature = "tiktoken", feature = "huggingface"))] +use litellm_error::Report; use litellm_host_python::{enter_native, release_gil}; #[cfg(feature = "fast")] use litellm_token_counter::fast::{FastCounter, FastTokenizer}; @@ -199,7 +201,7 @@ impl Tokenizer { let tokenizer = self.tiktoken()?; let text = self.text(text)?; release_gil(py, || tokenizer.encode_special(&text, &allowed)) - .map_err(PyRuntimeError::new_err) + .map_err(|error| PyRuntimeError::new_err(Report::new(&error).to_string())) } /// tiktoken's `encode_with_unstable`: `(stable_tokens, completions)`. @@ -258,8 +260,8 @@ impl Tokenizer { fn decode_bytes<'py>(&self, py: Python<'py>, ids: Vec) -> PyResult> { enter_native()?; let tokenizer = self.tiktoken()?; - let bytes = - release_gil(py, || tokenizer.decode_bytes(&ids)).map_err(PyKeyError::new_err)?; + let bytes = release_gil(py, || tokenizer.decode_bytes(&ids)) + .map_err(|error| PyKeyError::new_err(Report::new(&error).to_string()))?; Ok(PyBytes::new(py, &bytes)) } @@ -564,7 +566,7 @@ impl HuggingFaceEncoding { fn new(json: Option<&str>) -> PyResult { let inner = match json { Some(json) => encoding_from_json(json) - .map_err(|error| PyValueError::new_err(error.to_string()))?, + .map_err(|error| PyValueError::new_err(Report::new(&error).to_string()))?, None => Encoding::default(), }; Ok(Self { inner }) @@ -586,7 +588,7 @@ impl HuggingFaceEncoding { py: Python<'py>, ) -> PyResult<(Bound<'py, pyo3::types::PyType>, (String,))> { let json = encoding_to_json(&self.inner) - .map_err(|error| PyValueError::new_err(error.to_string()))?; + .map_err(|error| PyValueError::new_err(Report::new(&error).to_string()))?; Ok((py.get_type::(), (json,))) } diff --git a/litellm-rust/crates/python-compat/src/error.rs b/litellm-rust/crates/python-compat/src/error.rs index 43990eb7602..328c54aa934 100644 --- a/litellm-rust/crates/python-compat/src/error.rs +++ b/litellm-rust/crates/python-compat/src/error.rs @@ -10,8 +10,10 @@ pub enum Error { Unhashable(&'static str), #[error("value nests deeper than {MAX_DEPTH} levels")] TooDeep, - #[error("invalid pickle: {0}")] - InvalidPickle(String), + #[error("invalid pickle")] + InvalidPickle(#[source] serde_pickle::Error), + #[error("could not write pickle")] + PickleWrite(#[source] serde_pickle::Error), #[error("Object of type {0} is not JSON serializable")] NotJsonSerializable(&'static str), #[error("keys must be str, int, float, bool or None, not {0}")] diff --git a/litellm-rust/crates/python-compat/src/pickle.rs b/litellm-rust/crates/python-compat/src/pickle.rs index bff7442dc6b..78161d33a73 100644 --- a/litellm-rust/crates/python-compat/src/pickle.rs +++ b/litellm-rust/crates/python-compat/src/pickle.rs @@ -22,10 +22,8 @@ pub fn loads(data: &[u8]) -> Result { let mut deserializer = serde_pickle::Deserializer::new(data, DeOptions::new()); let value = Seed { depth: 0 } .deserialize(&mut deserializer) - .map_err(|error| Error::InvalidPickle(error.to_string()))?; - deserializer - .end() - .map_err(|error| Error::InvalidPickle(error.to_string()))?; + .map_err(Error::InvalidPickle)?; + deserializer.end().map_err(Error::InvalidPickle)?; Ok(value) } @@ -33,8 +31,7 @@ pub fn loads(data: &[u8]) -> Result { /// default. Sets and complex numbers are rejected rather than silently changing type. pub fn dumps(value: &Value) -> Result, Error> { check_picklable(value, 0)?; - serde_pickle::to_vec(&Pickled(value), SerOptions::new()) - .map_err(|error| Error::InvalidPickle(error.to_string())) + serde_pickle::to_vec(&Pickled(value), SerOptions::new()).map_err(Error::PickleWrite) } fn check_picklable(value: &Value, depth: usize) -> Result<(), Error> { diff --git a/litellm-rust/crates/secrets-aws/src/auth.rs b/litellm-rust/crates/secrets-aws/src/auth.rs index 06019cc3c9d..66f4f51bda9 100644 --- a/litellm-rust/crates/secrets-aws/src/auth.rs +++ b/litellm-rust/crates/secrets-aws/src/auth.rs @@ -79,9 +79,7 @@ impl ProvideCredentials for Credentials { self.auth .resolve_credentials(self.config.clone(), &|name| self.environment.get(name)) .await - .map_err(|_| { - CredentialsError::provider_error("secret manager authentication failed") - }) + .map_err(CredentialsError::provider_error) }) } } diff --git a/litellm-rust/crates/secrets-aws/src/error.rs b/litellm-rust/crates/secrets-aws/src/error.rs index c8ca671e9b5..c537da204dd 100644 --- a/litellm-rust/crates/secrets-aws/src/error.rs +++ b/litellm-rust/crates/secrets-aws/src/error.rs @@ -11,7 +11,7 @@ pub enum Error { #[error("KMS response has no plaintext")] MissingPlaintext, #[error("AWS request timed out")] - Timeout, + Timeout(#[source] #[redact] Box>), #[error("AWS KMS decrypt failed")] Decrypt(#[from] #[redact] Box>), #[error("AWS Secrets Manager read failed")] @@ -33,7 +33,7 @@ pub enum Error { #[error("AWS Secrets Manager response has no string payload")] MissingString, #[error("primary secret is not a JSON object")] - PrimarySecret, + PrimarySecret(#[source] #[redact] Option), #[error(transparent)] Operation(#[from] litellm_secrets_types::Error), } diff --git a/litellm-rust/crates/secrets-aws/src/secret_manager/read.rs b/litellm-rust/crates/secrets-aws/src/secret_manager/read.rs index b7583f9785f..d7f10a9847e 100644 --- a/litellm-rust/crates/secrets-aws/src/secret_manager/read.rs +++ b/litellm-rust/crates/secrets-aws/src/secret_manager/read.rs @@ -72,7 +72,7 @@ impl AwsSecretsManagerV2 { } let client = self.client_for_context(context)?; let value = match Self::read_with_client(&client, name, ReadPolicy::Python).await { - Err(Error::Read(_) | Error::MissingString | Error::Timeout) => None, + Err(Error::Read(_) | Error::MissingString | Error::Timeout(_)) => None, result => result?, }; Ok(PythonSecretRead::Value(value.map(Secret::String))) @@ -124,9 +124,10 @@ impl AwsSecretsManagerV2 { Self::read_with_client(&self.client, name, policy).await, policy, ) { - (Err(Error::Read(_) | Error::MissingString | Error::Timeout), ReadPolicy::Python) => { - Ok(None) - } + ( + Err(Error::Read(_) | Error::MissingString | Error::Timeout(_)), + ReadPolicy::Python, + ) => Ok(None), (result, _) => result, } } @@ -173,7 +174,7 @@ impl AwsSecretsManagerV2 { aws_sdk_secretsmanager::error::SdkError::TimeoutError(_) ) || matches!(&error, aws_sdk_secretsmanager::error::SdkError::DispatchFailure(failure) if failure.is_timeout()) => { - Err(Error::Timeout) + Err(Error::Timeout(Box::new(error))) } Err(error) if error @@ -216,9 +217,9 @@ fn resolve_payload(payload: PythonSecretRead, name: &str) -> Result Ok(value), PythonSecretRead::PrimaryJson(document) => { - let object: Value = - serde_json::from_str(document.expose()).map_err(|_| Error::PrimarySecret)?; - let object = object.as_object().ok_or(Error::PrimarySecret)?; + let object: Value = serde_json::from_str(document.expose()) + .map_err(|error| Error::PrimarySecret(Some(error)))?; + let object = object.as_object().ok_or(Error::PrimarySecret(None))?; Ok(object.get(name).cloned().map(Secret::from_json)) } } diff --git a/litellm-rust/crates/secrets-aws/tests/secret_manager/reads.rs b/litellm-rust/crates/secrets-aws/tests/secret_manager/reads.rs index 3c39924f09f..30aec467543 100644 --- a/litellm-rust/crates/secrets-aws/tests/secret_manager/reads.rs +++ b/litellm-rust/crates/secrets-aws/tests/secret_manager/reads.rs @@ -88,7 +88,7 @@ async fn failed_read_returns_none_but_invalid_primary_json_is_an_error( manager .read_secret_for_resolver("KEY", Some("invalid"), &|_: &str| None) .await, - Err(Error::PrimarySecret) + Err(Error::PrimarySecret(_)) )); assert!(matches!( manager.async_read_secret("no-string").await, @@ -143,7 +143,7 @@ async fn trait_read_applies_the_aws_operation_timeout() { }; assert!(matches!( BaseSecretManager::async_read_secret(&loaded_manager(&server), "key", &context).await, - Err(Error::Timeout) + Err(Error::Timeout(_)) )); } @@ -169,7 +169,7 @@ async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() { let manager = AwsSecretsManagerV2::new(Client::from_conf(config), Default::default()); assert!(matches!( manager.async_read_secret("key").await, - Err(Error::Timeout) + Err(Error::Timeout(_)) )); } diff --git a/litellm-rust/crates/secrets-azure/Cargo.toml b/litellm-rust/crates/secrets-azure/Cargo.toml index 7ec03fb98da..2aff6129c37 100644 --- a/litellm-rust/crates/secrets-azure/Cargo.toml +++ b/litellm-rust/crates/secrets-azure/Cargo.toml @@ -15,6 +15,7 @@ litellm-core-utils.workspace = true reqwest.workspace = true serde.workspace = true thiserror.workspace = true +url.workspace = true veil.workspace = true percent-encoding = "2.3" diff --git a/litellm-rust/crates/secrets-azure/src/error.rs b/litellm-rust/crates/secrets-azure/src/error.rs index 2862f08a56c..097f42cb89c 100644 --- a/litellm-rust/crates/secrets-azure/src/error.rs +++ b/litellm-rust/crates/secrets-azure/src/error.rs @@ -7,7 +7,7 @@ pub enum Error { #[error("{0} environment variable is missing")] MissingEnvironment(&'static str), #[error("AZURE_KEY_VAULT_URI is not a valid https vault URL")] - VaultUri, + VaultUri(#[source] Option), #[error("Azure Key Vault credentials are not configured")] MissingCredentials, #[error(transparent)] @@ -20,10 +20,26 @@ pub enum Error { Http( #[source] #[redact] - reqwest::Error, + litellm_http::Error, ), #[error("Azure Key Vault returned HTTP {0}")] Status(u16), #[error("Azure Key Vault response is missing the secret value")] MissingValue, } + +impl Error { + pub fn is_missing_value(&self) -> bool { + match self { + Self::MissingValue => true, + Self::Operation(_) + | Self::Timeout + | Self::MissingEnvironment(_) + | Self::VaultUri(_) + | Self::MissingCredentials + | Self::Auth(_) + | Self::Http(_) + | Self::Status(_) => false, + } + } +} diff --git a/litellm-rust/crates/secrets-azure/src/key_vault.rs b/litellm-rust/crates/secrets-azure/src/key_vault.rs index 095c451927c..d97b7a22cfc 100644 --- a/litellm-rust/crates/secrets-azure/src/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/src/key_vault.rs @@ -38,7 +38,7 @@ impl AzureKeyVault { environment: Arc, ) -> Result { if vault.host_str().is_none() { - return Err(Error::VaultUri); + return Err(Error::VaultUri(None)); } let inputs = AzureAuthInputs { azure_scope: ConfigValue::Value(Sourced::new( @@ -64,9 +64,9 @@ impl AzureKeyVault { let value = environment .get(AZURE_KEY_VAULT_URI) .ok_or(Error::MissingEnvironment(AZURE_KEY_VAULT_URI))?; - let vault = reqwest::Url::parse(&value).map_err(|_| Error::VaultUri)?; + let vault = reqwest::Url::parse(&value).map_err(|error| Error::VaultUri(Some(error)))?; if vault.scheme() != "https" || vault.host_str().is_none() { - return Err(Error::VaultUri); + return Err(Error::VaultUri(None)); } Self::with_client(client, vault, environment) } @@ -95,7 +95,7 @@ impl AzureKeyVault { let url = self .vault .join(&format!("secrets/{encoded_name}?api-version={API_VERSION}")) - .map_err(|_| Error::VaultUri)?; + .map_err(|error| Error::VaultUri(Some(error)))?; let response = self .client .get(url) @@ -103,14 +103,17 @@ impl AzureKeyVault { .header(reqwest::header::ACCEPT, "application/json") .send() .await - .map_err(Error::Http)?; + .map_err(|error| Error::Http(error.into()))?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(None); } if response.status() != reqwest::StatusCode::OK { return Err(Error::Status(response.status().as_u16())); } - let payload: SecretResponse = response.json().await.map_err(Error::Http)?; + let payload: SecretResponse = response + .json() + .await + .map_err(|error| Error::Http(error.into()))?; let value = payload.value.ok_or(Error::MissingValue)?; Ok(Some(SecretValue::new(value))) } diff --git a/litellm-rust/crates/secrets-azure/tests/key_vault.rs b/litellm-rust/crates/secrets-azure/tests/key_vault.rs index 05f62d25dce..a0ba9e6ddf8 100644 --- a/litellm-rust/crates/secrets-azure/tests/key_vault.rs +++ b/litellm-rust/crates/secrets-azure/tests/key_vault.rs @@ -122,13 +122,14 @@ async fn missing_value_is_an_error() { } #[rstest] -#[case::missing(None, true)] -#[case::http(Some("http://vault.example"), false)] -#[case::relative(Some("vault.example"), false)] -#[case::malformed(Some("://"), false)] +#[case::missing(None, true, false)] +#[case::http(Some("http://vault.example"), false, false)] +#[case::relative(Some("vault.example"), false, true)] +#[case::malformed(Some("://"), false, true)] fn new_validates_vault_environment( #[case] uri: Option<&'static str>, #[case] missing_environment: bool, + #[case] unparsable: bool, ) { let result = AzureKeyVault::new( litellm_http::Client::for_test(), @@ -145,7 +146,14 @@ fn new_validates_vault_environment( Err(Error::MissingEnvironment("AZURE_KEY_VAULT_URI")) )); } else { - assert!(matches!(result, Err(Error::VaultUri))); + let error = result.err().expect("invalid vault URI"); + assert!(matches!(error, Error::VaultUri(_))); + assert_eq!( + std::error::Error::source(&error) + .and_then(|cause| cause.downcast_ref::()) + .is_some(), + unparsable + ); } } diff --git a/litellm-rust/crates/secrets-cyberark/Cargo.toml b/litellm-rust/crates/secrets-cyberark/Cargo.toml index f630d5857d8..2ae2c1ea5c3 100644 --- a/litellm-rust/crates/secrets-cyberark/Cargo.toml +++ b/litellm-rust/crates/secrets-cyberark/Cargo.toml @@ -14,6 +14,7 @@ moka.workspace = true reqwest.workspace = true serde_json.workspace = true thiserror.workspace = true +url.workspace = true veil.workspace = true litellm-tracing.workspace = true percent-encoding = "2.3" diff --git a/litellm-rust/crates/secrets-cyberark/src/error.rs b/litellm-rust/crates/secrets-cyberark/src/error.rs index 72ca3416e28..48acf840be6 100644 --- a/litellm-rust/crates/secrets-cyberark/src/error.rs +++ b/litellm-rust/crates/secrets-cyberark/src/error.rs @@ -6,7 +6,7 @@ pub enum Error { Http( #[from] #[redact] - reqwest::Error, + litellm_http::Error, ), #[error("CyberArk Conjur authentication returned HTTP {0}")] AuthStatus(u16), @@ -17,13 +17,21 @@ pub enum Error { )] MissingCredentials, #[error("CyberArk client certificate could not be loaded")] - ClientCertificate, + ClientCertificate( + #[source] + #[redact] + litellm_http::Error, + ), #[error("CyberArk Conjur HTTP client could not be built")] - Client(#[redact] litellm_http::Error), + Client( + #[source] + #[redact] + litellm_http::Error, + ), #[error("invalid refresh interval")] - RefreshInterval, + RefreshInterval(#[source] std::num::ParseIntError), #[error("invalid CyberArk Conjur endpoint")] - Endpoint, + Endpoint(#[source] url::ParseError), #[error("CyberArk secret manager requires an enterprise license")] EnterpriseRequired, #[error(transparent)] diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs index 44473c9fe90..f1a6a77383f 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager.rs @@ -93,7 +93,7 @@ impl CyberArkSecretManager { let encoded = utf8_percent_encode(name, SECRET_NAME_SAFE); self.endpoint .join(&format!("secrets/{}/variable/{}", self.account, encoded)) - .map_err(|_| Error::Endpoint) + .map_err(Error::Endpoint) } } diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs index 6c31ff73498..1cda0ae5306 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager/client.rs @@ -67,15 +67,15 @@ impl CyberArkSecretManager { let client = pool .client(&config, ClientVariant::Provider) .map_err(|error| match error.tls_source() { - Some(TlsSource::ClientIdentity) => Error::ClientCertificate, - _ => Error::Client(error), + Some(TlsSource::ClientIdentity) => Error::ClientCertificate(error), + Some(TlsSource::CaBundle) | None => Error::Client(error), })?; let endpoint = reqwest::Url::parse( &environment .get(CYBERARK_API_BASE) .unwrap_or_else(|| DEFAULT_API_BASE.to_owned()), ) - .map_err(|_| Error::Endpoint)?; + .map_err(Error::Endpoint)?; let account = environment .get(CYBERARK_ACCOUNT) .unwrap_or_else(|| DEFAULT_ACCOUNT.to_owned()); @@ -88,7 +88,7 @@ impl CyberArkSecretManager { value .parse::() .map(Duration::from_secs) - .map_err(|_| Error::RefreshInterval) + .map_err(Error::RefreshInterval) }) .transpose()?; Ok(Self::with_client( @@ -108,7 +108,7 @@ impl CyberArkSecretManager { self.account, utf8_percent_encode(&self.username, SECRET_NAME_SAFE) )) - .map_err(|_| Error::Endpoint) + .map_err(Error::Endpoint) } pub(super) async fn authenticate( @@ -128,11 +128,14 @@ impl CyberArkSecretManager { context, ) .send() - .await?; + .await + .map_err(litellm_http::Error::from)?; if !response.status().is_success() { return Err(Error::AuthStatus(response.status().as_u16())); } - let token = SecretValue::new(STANDARD.encode(response.text().await?)); + let token = SecretValue::new( + STANDARD.encode(response.text().await.map_err(litellm_http::Error::from)?), + ); self.token.insert((), token.clone()).await; Ok(token) } diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager/read.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager/read.rs index daa9ed1114d..2862c6a956d 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager/read.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager/read.rs @@ -76,7 +76,8 @@ impl CyberArkSecretManager { context, ) .send() - .await?; + .await + .map_err(litellm_http::Error::from)?; let response = if matches!(retry, AuthenticationRetry::Unauthorized) && had_cached_token && response.status() == reqwest::StatusCode::UNAUTHORIZED @@ -89,7 +90,8 @@ impl CyberArkSecretManager { context, ) .send() - .await? + .await + .map_err(litellm_http::Error::from)? } else { response }; @@ -99,7 +101,7 @@ impl CyberArkSecretManager { if !response.status().is_success() { return Err(Error::Status(response.status().as_u16())); } - let value = SecretValue::new(response.text().await?); + let value = SecretValue::new(response.text().await.map_err(litellm_http::Error::from)?); Ok(Some(value)) } } diff --git a/litellm-rust/crates/secrets-cyberark/src/secret_manager/write.rs b/litellm-rust/crates/secrets-cyberark/src/secret_manager/write.rs index 87f6cea3619..34a5f4ccc32 100644 --- a/litellm-rust/crates/secrets-cyberark/src/secret_manager/write.rs +++ b/litellm-rust/crates/secrets-cyberark/src/secret_manager/write.rs @@ -83,7 +83,9 @@ impl CyberArkSecretManager { ) .send() .await - .map_err(|source| WriteFailure::request(source.into(), url.clone())) + .map_err(|source| { + WriteFailure::request(litellm_http::Error::from(source).into(), url.clone()) + }) } pub(super) async fn ensure_variable_exists( diff --git a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs index a3c20a0db3d..8a0746f8002 100644 --- a/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs +++ b/litellm-rust/crates/secrets-cyberark/tests/secret_manager/configuration.rs @@ -201,7 +201,7 @@ async fn trait_read_applies_cyberark_operation_timeout_to_authentication() { match result { Err(Error::Timeout) => {} - Err(Error::Http(error)) => assert!(error.is_timeout()), + Err(Error::Http(error)) => assert_eq!(error.status(), Some(408)), other => panic!("expected timeout, got {other:?}"), } } @@ -237,7 +237,7 @@ fn new_validates_credentials_before_license_and_configuration() { }), true ), - Err(Error::RefreshInterval) + Err(Error::RefreshInterval(_)) )); assert!(matches!( from_environment( @@ -248,7 +248,7 @@ fn new_validates_credentials_before_license_and_configuration() { }), true ), - Err(Error::Endpoint) + Err(Error::Endpoint(_)) )); } @@ -263,7 +263,7 @@ fn certificate_only_credentials_are_validated_as_a_client_identity() { true, ); - assert!(matches!(result, Err(Error::ClientCertificate))); + assert!(matches!(result, Err(Error::ClientCertificate(_)))); } #[rstest] @@ -347,7 +347,7 @@ fn invalid_client_identity_is_not_ignored( true, ); - assert!(matches!(result, Err(Error::ClientCertificate))); + assert!(matches!(result, Err(Error::ClientCertificate(_)))); } #[rstest] @@ -413,7 +413,7 @@ fn new_reports_missing_client_certificate_files() { }), true ), - Err(Error::ClientCertificate) + Err(Error::ClientCertificate(_)) )); } diff --git a/litellm-rust/crates/secrets-google/Cargo.toml b/litellm-rust/crates/secrets-google/Cargo.toml index 3ce14fe7a12..06b3747def5 100644 --- a/litellm-rust/crates/secrets-google/Cargo.toml +++ b/litellm-rust/crates/secrets-google/Cargo.toml @@ -17,6 +17,7 @@ base64.workspace = true crc32c = "0.6.8" serde_json.workspace = true thiserror.workspace = true +url.workspace = true veil.workspace = true google-cloud-kms-v1 = "1.14.0" google-cloud-gax = { version = "1.14.0", default-features = false } diff --git a/litellm-rust/crates/secrets-google/src/error.rs b/litellm-rust/crates/secrets-google/src/error.rs index 159e4983ddc..486c7a3f35e 100644 --- a/litellm-rust/crates/secrets-google/src/error.rs +++ b/litellm-rust/crates/secrets-google/src/error.rs @@ -26,7 +26,7 @@ pub enum Error { Http( #[from] #[redact] - reqwest::Error, + litellm_http::Error, ), #[error("Google Secret Manager returned HTTP {0}")] Status(u16), @@ -35,15 +35,41 @@ pub enum Error { #[error("required environment variable is missing: {0}")] MissingEnvironment(&'static str), #[error("invalid refresh interval")] - RefreshInterval, + RefreshInterval(#[source] std::num::ParseIntError), #[error("payload is not valid base64")] - Base64(#[from] base64::DecodeError), + Base64( + #[from] + #[redact] + base64::DecodeError, + ), #[error("Google Secret Manager payload checksum mismatch")] - Checksum, + Checksum(#[source] Option), #[error("decrypted value is not UTF-8")] - Utf8, + Utf8(#[source] std::str::Utf8Error), #[error("invalid Google Secret Manager endpoint")] - Endpoint, + Endpoint(#[source] url::ParseError), #[error("Google Secret Manager requires an enterprise license")] EnterpriseRequired, } + +impl Error { + pub fn is_not_found(&self) -> bool { + match self { + Self::Status(status) => *status == 404, + Self::Operation(_) + | Self::Timeout + | Self::Client(_) + | Self::Auth(_) + | Self::Kms(_) + | Self::Http(_) + | Self::MissingPayload + | Self::MissingEnvironment(_) + | Self::RefreshInterval(_) + | Self::Base64(_) + | Self::Checksum(_) + | Self::Utf8(_) + | Self::Endpoint(_) + | Self::EnterpriseRequired => false, + } + } +} diff --git a/litellm-rust/crates/secrets-google/src/secret_manager.rs b/litellm-rust/crates/secrets-google/src/secret_manager.rs index 08ba466b799..463d1f6fd73 100644 --- a/litellm-rust/crates/secrets-google/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-google/src/secret_manager.rs @@ -92,12 +92,12 @@ impl GoogleSecretManager { let ttl = environment .get(GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL) .filter(|v| !v.is_empty()) - .map(|v| v.parse::().map_err(|_| Error::RefreshInterval)) + .map(|v| v.parse::().map_err(Error::RefreshInterval)) .transpose()? .unwrap_or( environment .get(SECRET_MANAGER_REFRESH_INTERVAL) - .map(|v| v.parse::().map_err(|_| Error::RefreshInterval)) + .map(|v| v.parse::().map_err(Error::RefreshInterval)) .transpose()? .unwrap_or(DEFAULT_REFRESH_INTERVAL.as_secs() as i64), ); @@ -164,20 +164,21 @@ impl GoogleSecretManager { ), percent_encoding::utf8_percent_encode(name, percent_encoding::NON_ALPHANUMERIC) )) - .map_err(|_| Error::Endpoint)?; + .map_err(Error::Endpoint)?; let response = self .client .get(url) .headers(self.credentials.request_headers().await?) .send() - .await?; + .await + .map_err(litellm_http::Error::from)?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(None); } if response.status() != reqwest::StatusCode::OK { return Err(Error::Status(response.status().as_u16())); } - let response: Response = response.json().await?; + let response: Response = response.json().await.map_err(litellm_http::Error::from)?; let Some(payload) = response.payload else { return Err(Error::MissingPayload); }; @@ -186,12 +187,15 @@ impl GoogleSecretManager { }; let bytes = STANDARD.decode(data)?; if let Some(expected) = payload.data_crc32c { - let expected = expected.parse::().map_err(|_| Error::Checksum)?; + let expected = expected + .parse::() + .map_err(|error| Error::Checksum(Some(error)))?; if crc32c::crc32c(&bytes) != expected { - return Err(Error::Checksum); + return Err(Error::Checksum(None)); } } - let plaintext = String::from_utf8(bytes).map_err(|_| Error::Utf8)?; + let plaintext = + String::from_utf8(bytes).map_err(|error| Error::Utf8(error.utf8_error()))?; let value = SecretValue::new(plaintext); Ok(Some(value)) } diff --git a/litellm-rust/crates/secrets-google/tests/secret_manager.rs b/litellm-rust/crates/secrets-google/tests/secret_manager.rs index f2cb5ff3603..615e331cb96 100644 --- a/litellm-rust/crates/secrets-google/tests/secret_manager.rs +++ b/litellm-rust/crates/secrets-google/tests/secret_manager.rs @@ -152,8 +152,8 @@ async fn failed_or_missing_reads_are_not_cached( assert!(matches!(result, Err(Error::MissingPayload))); } ExpectedReadFailure::Base64 => assert!(matches!(result, Err(Error::Base64(_)))), - ExpectedReadFailure::Utf8 => assert!(matches!(result, Err(Error::Utf8))), - ExpectedReadFailure::Checksum => assert!(matches!(result, Err(Error::Checksum))), + ExpectedReadFailure::Utf8 => assert!(matches!(result, Err(Error::Utf8(_)))), + ExpectedReadFailure::Checksum => assert!(matches!(result, Err(Error::Checksum(None)))), } drop(failing); Mock::given(path( @@ -245,7 +245,7 @@ fn google_manager_rejects_invalid_refresh_intervals(#[case] variable: &'static s assert!(matches!( GoogleSecretManager::new(litellm_http::Client::for_test(), environment, true), - Err(Error::RefreshInterval) + Err(Error::RefreshInterval(_)) )); } diff --git a/litellm-rust/crates/secrets-hashicorp/Cargo.toml b/litellm-rust/crates/secrets-hashicorp/Cargo.toml index 7dd3d3c674f..a50718c9574 100644 --- a/litellm-rust/crates/secrets-hashicorp/Cargo.toml +++ b/litellm-rust/crates/secrets-hashicorp/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-error.workspace = true litellm-core-utils.workspace = true litellm-secrets-types.workspace = true rustify.workspace = true diff --git a/litellm-rust/crates/secrets-hashicorp/src/config.rs b/litellm-rust/crates/secrets-hashicorp/src/config.rs index d32491199c4..4232667ae16 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/config.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/config.rs @@ -153,9 +153,11 @@ fn refresh_interval(environment: &dyn Lookup) -> Result { let Some(value) = value else { return Ok(DEFAULT_REFRESH_INTERVAL); }; - let seconds: i64 = value.parse().map_err(|_| Error::RefreshInterval)?; + let seconds: i64 = value + .parse() + .map_err(|error| Error::RefreshInterval(Some(error)))?; if seconds < 0 { - return Err(Error::RefreshInterval); + return Err(Error::RefreshInterval(None)); } Ok(Duration::from_secs(seconds as u64)) } diff --git a/litellm-rust/crates/secrets-hashicorp/src/error.rs b/litellm-rust/crates/secrets-hashicorp/src/error.rs index 1813cb486bc..87b0e5f5aa2 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/error.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/error.rs @@ -3,7 +3,7 @@ pub enum Error { #[error("HashiCorp Vault requires an enterprise license")] EnterpriseRequired, #[error("invalid secret name")] - InvalidSecretName(litellm_secrets_types::Error), + InvalidSecretName(#[source] litellm_secrets_types::Error), #[error(transparent)] Operation(#[from] litellm_secrets_types::Error), #[error("HashiCorp Vault client failed")] @@ -12,31 +12,39 @@ pub enum Error { #[redact] vaultrs::error::ClientError, ), - #[error("HashiCorp Vault client settings are invalid: {message}")] - ClientSettings { message: String }, - #[error("HashiCorp Vault TLS identity could not be configured for {path}: {message}")] + #[error("HashiCorp Vault client settings are invalid")] + ClientSettings(#[source] vaultrs::client::VaultClientSettingsBuilderError), + #[error("HashiCorp Vault TLS identity could not be configured for {path}")] TlsIdentity { path: std::path::PathBuf, - message: String, + source: litellm_error::Source, }, #[error("HashiCorp Vault login returned HTTP {status}")] LoginStatus { status: u16 }, #[error("HashiCorp Vault login response is malformed")] - MalformedLogin, + MalformedLogin( + #[source] + #[redact] + vaultrs::error::ClientError, + ), #[error("HashiCorp Vault authentication is not configured")] NoAuthConfigured, #[error("HashiCorp Vault returned HTTP {status}")] Status { status: u16 }, #[error("HashiCorp Vault response payload is malformed")] - MalformedPayload, + MalformedPayload( + #[source] + #[redact] + vaultrs::error::ClientError, + ), #[error("HashiCorp Vault secret value is not a string")] NonStringValue, #[error("HashiCorp Vault data key conflicts with description")] DataKeyConflictsWithDescription, #[error("HashiCorp Vault secret version exceeds CAS range")] - CasVersionOverflow, + CasVersionOverflow(#[source] std::num::TryFromIntError), #[error("HashiCorp Vault operation timed out")] Timeout, #[error("invalid HashiCorp Vault refresh interval")] - RefreshInterval, + RefreshInterval(#[source] Option), } diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs index f0d4fe8c0b4..3db5bf9b541 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager.rs @@ -178,20 +178,22 @@ fn map_api_error(error: ClientError, context: ErrorContext) -> Error { ErrorContext::Login => Error::LoginStatus { status: code }, ErrorContext::Read | ErrorContext::Secret => Error::Status { status: code }, }, - ClientError::JsonParseError { source } => match context { - ErrorContext::Login => Error::MalformedLogin, - ErrorContext::Read => Error::MalformedPayload, - ErrorContext::Secret => Error::Client(ClientError::JsonParseError { source }), + error @ ClientError::JsonParseError { .. } => match context { + ErrorContext::Login => Error::MalformedLogin(error), + ErrorContext::Read => Error::MalformedPayload(error), + ErrorContext::Secret => Error::Client(error), }, - ClientError::ResponseEmptyError | ClientError::ResponseDataEmptyError => { - malformed_response(context) + error @ (ClientError::ResponseEmptyError | ClientError::ResponseDataEmptyError) => { + malformed_response(context, error) } ClientError::RestClientError { source } => match source { RustifyClientError::ServerResponseError { code, .. } => match context { ErrorContext::Login => Error::LoginStatus { status: code }, ErrorContext::Read | ErrorContext::Secret => Error::Status { status: code }, }, - RustifyClientError::ResponseParseError { .. } => malformed_response(context), + source @ RustifyClientError::ResponseParseError { .. } => { + malformed_response(context, ClientError::RestClientError { source }) + } source => Error::Client(ClientError::RestClientError { source }), }, error => Error::Client(error), @@ -208,10 +210,10 @@ fn api_status(error: &ClientError) -> Option { } } -fn malformed_response(context: ErrorContext) -> Error { +fn malformed_response(context: ErrorContext, error: ClientError) -> Error { match context { - ErrorContext::Login => Error::MalformedLogin, - ErrorContext::Read => Error::MalformedPayload, - ErrorContext::Secret => Error::Client(ClientError::ResponseDataEmptyError), + ErrorContext::Login => Error::MalformedLogin(error), + ErrorContext::Read => Error::MalformedPayload(error), + ErrorContext::Secret => Error::Client(error), } } diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager/client.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager/client.rs index c1dc073dcf8..70ecc35f6e6 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager/client.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager/client.rs @@ -83,9 +83,7 @@ impl HashicorpVault { .ca_certs(Vec::new()) .verify(true) .build() - .map_err(|message| Error::ClientSettings { - message: message.to_string(), - })?; + .map_err(Error::ClientSettings)?; VaultClient::new(settings).map_err(Error::Client) } } @@ -94,16 +92,16 @@ fn identity_for(tls: Option<&TlsCertAuth>) -> Result, Error> { tls.map(|tls| { let cert: Vec = std::fs::read(&tls.cert_path).map_err(|source| Error::TlsIdentity { path: tls.cert_path.clone(), - message: source.to_string(), + source: litellm_error::Source::new(source), })?; let key: Vec = std::fs::read(&tls.key_path).map_err(|source| Error::TlsIdentity { path: tls.key_path.clone(), - message: source.to_string(), + source: litellm_error::Source::new(source), })?; Identity::from_pem(&[cert.as_slice(), key.as_slice()].concat()).map_err(|source| { Error::TlsIdentity { path: tls.cert_path.clone(), - message: source.to_string(), + source: litellm_error::Source::new(source), } }) }) diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager/raw.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager/raw.rs index 0325561a7a0..a74beee811b 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager/raw.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager/raw.rs @@ -4,14 +4,17 @@ use vaultrs::api::kv2::requests::{ DeleteLatestSecretVersionRequest, ReadSecretRequest, SetSecretRequest, }; -#[derive(veil::Redact)] +#[derive(thiserror::Error, veil::Redact)] pub enum RawOperationError { + #[error(transparent)] Local(Error), + #[error("HashiCorp Vault authentication failed")] Authentication { source: Error, url: String, certificate: bool, }, + #[error("HashiCorp Vault {method} request returned HTTP {status}")] Http { method: String, url: String, @@ -19,11 +22,14 @@ pub enum RawOperationError { #[redact] body: Vec, }, - Transport(#[redact] RustifyClientError), - Timeout { - method: String, - elapsed: Duration, - }, + #[error("HashiCorp Vault request failed")] + Transport( + #[source] + #[redact] + RustifyClientError, + ), + #[error("HashiCorp Vault {method} request timed out")] + Timeout { method: String, elapsed: Duration }, } impl HashicorpVault { diff --git a/litellm-rust/crates/secrets-hashicorp/src/secret_manager/write.rs b/litellm-rust/crates/secrets-hashicorp/src/secret_manager/write.rs index 0b1acb3a179..15a0e4c7dff 100644 --- a/litellm-rust/crates/secrets-hashicorp/src/secret_manager/write.rs +++ b/litellm-rust/crates/secrets-hashicorp/src/secret_manager/write.rs @@ -37,7 +37,7 @@ impl HashicorpVault { .await { Ok(metadata) => u32::try_from(metadata.current_version) - .map_err(|_| Error::CasVersionOverflow)?, + .map_err(Error::CasVersionOverflow)?, Err(error) if api_status(&error) == Some(404) => 0, Err(_) => return Err(map_api_error(error, ErrorContext::Secret)), }; diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/configuration.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/configuration.rs index be3059ce986..b45b843fde8 100644 --- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/configuration.rs +++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/configuration.rs @@ -32,7 +32,7 @@ fn invalid_refresh_intervals_are_rejected(#[case] value: &str) { assert!(matches!( HashicorpVaultConfig::from_environment(environment.as_ref()), - Err(Error::RefreshInterval) + Err(Error::RefreshInterval(_)) )); } @@ -79,6 +79,39 @@ async fn approle_login_uses_namespace_and_reuses_the_token() { assert!(manager.async_read_secret("name-2").await.unwrap().is_none()); } +#[rstest] +#[tokio::test] +async fn an_unreadable_client_certificate_keeps_its_io_cause() { + let server: MockServer = MockServer::start().await; + let directory: tempfile::TempDir = tempfile::tempdir().unwrap(); + let cert_path = directory.path().join("missing.crt"); + let values: HashMap = HashMap::from([ + ("HCP_VAULT_ADDR".to_owned(), server.uri()), + ( + "HCP_VAULT_CLIENT_CERT".to_owned(), + cert_path.to_str().unwrap().to_owned(), + ), + ( + "HCP_VAULT_CLIENT_KEY".to_owned(), + cert_path.to_str().unwrap().to_owned(), + ), + ]); + let environment: Arc = + Arc::new(move |name: &str| values.get(name).cloned()); + let manager: HashicorpVault = HashicorpVault::new(environment, true).unwrap(); + + let error = manager.async_read_secret("name").await.unwrap_err(); + + let cause = std::error::Error::source(&error) + .and_then(|cause| cause.downcast_ref::()) + .and_then(|cause| cause.downcast_ref::()) + .expect("io cause"); + assert_eq!(cause.kind(), std::io::ErrorKind::NotFound); + let report = litellm_error::Report::new(&error).to_string(); + assert_eq!(report.matches(cert_path.to_str().unwrap()).count(), 1); + assert!(report.ends_with(&cause.to_string())); +} + #[rstest] #[tokio::test] async fn tls_login_posts_the_role_and_uses_the_client_identity() { diff --git a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/reads.rs b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/reads.rs index 0c662eb5b55..58dcc17957c 100644 --- a/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/reads.rs +++ b/litellm-rust/crates/secrets-hashicorp/tests/secret_manager/reads.rs @@ -138,7 +138,7 @@ async fn read_responses_distinguish_absence_and_malformed_payloads( .await; match expected { ExpectedRead::Missing => assert!(result.unwrap().is_none()), - ExpectedRead::Malformed => assert!(matches!(result, Err(Error::MalformedPayload))), + ExpectedRead::Malformed => assert!(matches!(result, Err(Error::MalformedPayload(_)))), ExpectedRead::NonString => assert!(matches!(result, Err(Error::NonStringValue))), } } diff --git a/litellm-rust/crates/secrets-types/tests/cache.rs b/litellm-rust/crates/secrets-types/tests/cache.rs index cec4f5198e3..18efde4289b 100644 --- a/litellm-rust/crates/secrets-types/tests/cache.rs +++ b/litellm-rust/crates/secrets-types/tests/cache.rs @@ -110,7 +110,7 @@ async fn concurrent_misses_share_a_load_and_cancellation_allows_a_retry() { } #[tokio::test] -async fn refresh_bypasses_cached_values_and_errors_and_absence_are_retried() { +async fn refresh_bypasses_cached_values() { let cache = SecretCache::new(10, Duration::from_secs(60)); cache.insert("key", "old").await; assert_eq!( @@ -127,9 +127,21 @@ async fn refresh_bypasses_cached_values_and_errors_and_absence_are_retried() { .unwrap(), Some("fresh") ); - for result in [Err("failure"), Ok(None), Ok(Some("recovered"))] { - assert_eq!(cache.read("retry", async { result }).await, result); - } +} + +#[rstest] +#[case::failure(Err("failure"))] +#[case::absence(Ok(None))] +#[tokio::test] +async fn errors_and_absence_are_retried(#[case] first: Result, &'static str>) { + let cache = SecretCache::new(10, Duration::from_secs(60)); + assert_eq!(cache.read("retry", async { first }).await, first); + assert_eq!( + cache + .read("retry", async { Ok::<_, &str>(Some("recovered")) }) + .await, + Ok(Some("recovered")) + ); } #[tokio::test] diff --git a/litellm-rust/crates/secrets/Cargo.toml b/litellm-rust/crates/secrets/Cargo.toml index 3655ce8bbc2..023d61136e4 100644 --- a/litellm-rust/crates/secrets/Cargo.toml +++ b/litellm-rust/crates/secrets/Cargo.toml @@ -23,6 +23,7 @@ litellm-secrets-hashicorp = { workspace = true, optional = true } litellm-secrets-azure = { workspace = true, optional = true } litellm-secrets-cyberark = { workspace = true, optional = true } litellm-core-utils.workspace = true +litellm-error.workspace = true litellm-http.workspace = true base64.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/secrets/src/compatibility.rs b/litellm-rust/crates/secrets/src/compatibility.rs index 621f7a3f432..368c9931b51 100644 --- a/litellm-rust/crates/secrets/src/compatibility.rs +++ b/litellm-rust/crates/secrets/src/compatibility.rs @@ -40,11 +40,9 @@ pub async fn get_secret_from_python_manager( }; match result { #[cfg(feature = "google")] - Err(Error::Google(litellm_secrets_google::Error::Status(404))) => { - Err(Error::ManagedSecretMissing) - } + Err(Error::Google(error)) if error.is_not_found() => Err(Error::ManagedSecretMissing), #[cfg(feature = "azure")] - Err(Error::Azure(litellm_secrets_azure::Error::MissingValue)) => Ok(None), + Err(Error::Azure(error)) if error.is_missing_value() => Ok(None), Ok(None) if matches!(manager, SecretManager::External(_)) && manager.system() == KeyManagementSystem::AzureKeyVault => diff --git a/litellm-rust/crates/secrets/src/error.rs b/litellm-rust/crates/secrets/src/error.rs index 7fc756c5c7d..ade8555242d 100644 --- a/litellm-rust/crates/secrets/src/error.rs +++ b/litellm-rust/crates/secrets/src/error.rs @@ -7,9 +7,9 @@ pub enum Error { #[error("encrypted environment value is missing")] MissingCiphertext, #[error("ciphertext is not valid base64 for the configured manager")] - InvalidCiphertext, + InvalidCiphertext(#[source] Option), #[error("decrypted value is not UTF-8")] - Utf8, + Utf8(#[source] std::str::Utf8Error), #[error(transparent)] Client(#[from] litellm_http::Error), #[error("unsupported OIDC provider or missing build feature")] @@ -19,21 +19,21 @@ pub enum Error { #[error("OIDC environment variable is missing")] MissingEnvironment, #[error("OIDC request failed")] - OidcHttp, + OidcHttp(#[source] litellm_http::Error), #[error("OIDC provider returned HTTP {0}")] OidcStatus(u16), #[error("OIDC response is invalid")] - OidcResponse, + OidcResponse(#[source] litellm_http::Error), #[error("OIDC file path must be absolute and within the credential allowlist")] UnsafeOidcPath, #[error("OIDC file could not be read")] - OidcFile, + OidcFile(#[source] std::io::Error), #[error("secret cannot be converted to {expected}")] TypeMismatch { expected: &'static str }, #[error("external secret manager failed")] - ExternalManager(#[source] Box), + ExternalManager(#[source] litellm_error::Source), #[error("external secret manager read failed")] - ExternalRead(#[source] Box), + ExternalRead(#[source] litellm_error::Source), #[cfg(feature = "aws")] #[error(transparent)] Aws(#[from] litellm_secrets_aws::Error), diff --git a/litellm-rust/crates/secrets/src/handler.rs b/litellm-rust/crates/secrets/src/handler.rs index db37d793384..f42e09c6bdb 100644 --- a/litellm-rust/crates/secrets/src/handler.rs +++ b/litellm-rust/crates/secrets/src/handler.rs @@ -87,7 +87,8 @@ pub async fn get_secret_from_manager( let plaintext = client .decrypt(decode_ciphertext(&ciphertext, Base64Mode::Permissive)?) .await?; - let value = String::from_utf8(plaintext).map_err(|_| Error::Utf8)?; + let value = + String::from_utf8(plaintext).map_err(|error| Error::Utf8(error.utf8_error()))?; Ok(Some(Secret::String(SecretValue::new(value.trim())))) } #[cfg(feature = "google")] @@ -98,7 +99,8 @@ pub async fn get_secret_from_manager( let plaintext = client .decrypt(decode_ciphertext(&ciphertext, Base64Mode::Canonical)?) .await?; - let value = String::from_utf8(plaintext).map_err(|_| Error::Utf8)?; + let value = + String::from_utf8(plaintext).map_err(|error| Error::Utf8(error.utf8_error()))?; Ok(Some(Secret::String(SecretValue::new(value)))) } #[cfg(feature = "aws")] @@ -149,9 +151,9 @@ fn decode_ciphertext(value: &str, mode: Base64Mode) -> Result, Error> { }; let ciphertext = STANDARD .decode(&encoded) - .map_err(|_| Error::InvalidCiphertext)?; + .map_err(|error| Error::InvalidCiphertext(Some(error)))?; if canonical && STANDARD.encode(&ciphertext) != encoded { - return Err(Error::InvalidCiphertext); + return Err(Error::InvalidCiphertext(None)); } Ok(ciphertext) } diff --git a/litellm-rust/crates/secrets/src/oidc.rs b/litellm-rust/crates/secrets/src/oidc.rs index adeb6c3db00..24e6912c797 100644 --- a/litellm-rust/crates/secrets/src/oidc.rs +++ b/litellm-rust/crates/secrets/src/oidc.rs @@ -191,7 +191,7 @@ impl OidcResolver { .header("Accept", "application/json; api-version=2.0") .send() .await - .map_err(|_| Error::OidcHttp)?; + .map_err(|error| Error::OidcHttp(error.into()))?; if response.status() != reqwest::StatusCode::OK { return Err(Error::OidcStatus(response.status().as_u16())); } @@ -199,7 +199,10 @@ impl OidcResolver { struct Token { value: Option, } - let token: Token = response.json().await.map_err(|_| Error::OidcResponse)?; + let token: Token = response + .json() + .await + .map_err(|error| Error::OidcResponse(error.into()))?; if let Some(value) = &token.value { self.cache .insert( @@ -225,11 +228,14 @@ impl OidcResolver { .header("Metadata-Flavor", "Google") .send() .await - .map_err(|_| Error::OidcHttp)?; + .map_err(|error| Error::OidcHttp(error.into()))?; if response.status() != reqwest::StatusCode::OK { return Err(Error::OidcStatus(response.status().as_u16())); } - let token = response.text().await.map_err(|_| Error::OidcResponse)?; + let token = response + .text() + .await + .map_err(|error| Error::OidcResponse(error.into()))?; let now = (self.clock)(); let ttl = oidc_token_cache_ttl(&token, now, GOOGLE_TOKEN_MAX_TTL); let value = SecretValue::new(token); @@ -259,7 +265,7 @@ async fn read_file(path: &str) -> Result { tokio::fs::read_to_string(path) .await .map(|value| SecretValue::new(value.replace("\r\n", "\n").replace('\r', "\n"))) - .map_err(|_| Error::OidcFile) + .map_err(Error::OidcFile) } async fn read_allowed_file( @@ -271,7 +277,7 @@ async fn read_allowed_file( } let resolved = tokio::fs::canonicalize(path) .await - .map_err(|_| Error::OidcFile)?; + .map_err(Error::OidcFile)?; let allowed = environment .get(OIDC_ALLOWED_CREDENTIAL_DIRS) .filter(|v| !v.is_empty()) @@ -283,7 +289,7 @@ async fn read_allowed_file( return tokio::fs::read_to_string(&resolved) .await .map(|value| SecretValue::new(value.replace("\r\n", "\n").replace('\r', "\n"))) - .map_err(|_| Error::OidcFile); + .map_err(Error::OidcFile); } } Err(Error::UnsafeOidcPath) diff --git a/litellm-rust/crates/secrets/tests/aws.rs b/litellm-rust/crates/secrets/tests/aws.rs index f28f404c3ff..abad04f31c5 100644 --- a/litellm-rust/crates/secrets/tests/aws.rs +++ b/litellm-rust/crates/secrets/tests/aws.rs @@ -218,6 +218,6 @@ async fn aws_handler_reads_ciphertext_decodes_trims_and_redacts() { )); assert!(matches!( get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| Some("abc".into())).await, - Err(Error::InvalidCiphertext) + Err(Error::InvalidCiphertext(_)) )); } diff --git a/litellm-rust/crates/secrets/tests/google.rs b/litellm-rust/crates/secrets/tests/google.rs index fa1db9916e8..5542c864dfd 100644 --- a/litellm-rust/crates/secrets/tests/google.rs +++ b/litellm-rust/crates/secrets/tests/google.rs @@ -108,7 +108,7 @@ async fn google_handler_requires_canonical_base64_and_preserves_plaintext_whites STANDARD.encode("encrypted") ))) .await, - Err(Error::InvalidCiphertext) + Err(Error::InvalidCiphertext(_)) )); assert!(matches!( get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| None).await, diff --git a/litellm-rust/crates/secrets/tests/resolution.rs b/litellm-rust/crates/secrets/tests/resolution.rs index 0858089b94f..8435e051315 100644 --- a/litellm-rust/crates/secrets/tests/resolution.rs +++ b/litellm-rust/crates/secrets/tests/resolution.rs @@ -17,74 +17,81 @@ fn resolver(value: Option<&str>) -> SecretResolver { } #[rstest::rstest] -#[case::environment(false)] -#[case::manager(true)] +#[case::capitalized_true("True", Some(true))] +#[case::padded_false(" FALSE ", Some(false))] +#[case::empty("", None)] +#[case::parenthesized("(True)", None)] +#[case::json("{\"key\":1}", None)] #[tokio::test] -async fn native_reads_preserve_strings_and_report_conversion_errors(#[case] managed: bool) { - for raw in ["True", " FALSE ", "", "(True)", "{\"key\":1}"] { - let state = if managed { - SecretManagerState::new( - SecretManager::External(Arc::new(FixedManager::custom(Ok(Some(Secret::String( - SecretValue::new(raw), - )))))), - KeyManagementSettings::default(), - ) - } else { - SecretManagerState::default() - }; - let resolver = SecretResolver::new( - Arc::new(state), - Arc::new(move |_: &str| Some(raw.to_owned())), - OidcResolver::new(litellm_http::Client::for_test()), - ); - assert_eq!( - resolver - .get_secret_str("key", None) - .await - .unwrap() - .unwrap() - .expose(), - raw - ); - assert_eq!( - resolver.get_secret("key", None).await.unwrap(), - Some(Secret::String(SecretValue::new(raw))) - ); - if raw == "True" || raw == " FALSE " { - assert_eq!( - resolver.get_secret_bool("key", None).await.unwrap(), - Some(raw == "True") - ); - } else { - assert!(matches!( - resolver.get_secret_bool("key", None).await, - Err(Error::TypeMismatch { - expected: "boolean" - }) - )); - } +async fn native_reads_preserve_strings_and_report_conversion_errors( + #[case] raw: &'static str, + #[case] boolean: Option, + #[values(false, true)] managed: bool, +) { + let state = if managed { + SecretManagerState::new( + SecretManager::External(Arc::new(FixedManager::custom(Ok(Some(Secret::String( + SecretValue::new(raw), + )))))), + KeyManagementSettings::default(), + ) + } else { + SecretManagerState::default() + }; + let resolver = SecretResolver::new( + Arc::new(state), + Arc::new(move |_: &str| Some(raw.to_owned())), + OidcResolver::new(litellm_http::Client::for_test()), + ); + assert_eq!( + resolver + .get_secret_str("key", None) + .await + .unwrap() + .unwrap() + .expose(), + raw + ); + assert_eq!( + resolver.get_secret("key", None).await.unwrap(), + Some(Secret::String(SecretValue::new(raw))) + ); + match boolean { + Some(expected) => assert_eq!( + resolver.get_secret_bool("key", None).await.unwrap(), + Some(expected) + ), + None => assert!(matches!( + resolver.get_secret_bool("key", None).await, + Err(Error::TypeMismatch { + expected: "boolean" + }) + )), } } +#[rstest::rstest] +#[case::absent(Ok(None))] +#[case::failed(Err(()))] #[tokio::test] -async fn native_defaults_apply_to_absence_but_never_hide_provider_failures() { - for reply in [Ok(None), Err(())] { - let resolver = SecretResolver::new( - Arc::new(SecretManagerState::new( - SecretManager::External(Arc::new(FixedManager::custom(reply.clone()))), - KeyManagementSettings::default(), - )), - Arc::new(|_: &str| None), - OidcResolver::new(litellm_http::Client::for_test()), - ); - let result = resolver - .get_secret_str("key", Some(SecretValue::new("default"))) - .await; - match reply { - Ok(None) => assert_eq!(result.unwrap().unwrap().expose(), "default"), - Err(()) => assert!(matches!(result, Err(Error::MissingCiphertext))), - Ok(Some(_)) => unreachable!(), - } +async fn native_defaults_apply_to_absence_but_never_hide_provider_failures( + #[case] reply: Result, ()>, +) { + let resolver = SecretResolver::new( + Arc::new(SecretManagerState::new( + SecretManager::External(Arc::new(FixedManager::custom(reply.clone()))), + KeyManagementSettings::default(), + )), + Arc::new(|_: &str| None), + OidcResolver::new(litellm_http::Client::for_test()), + ); + let result = resolver + .get_secret_str("key", Some(SecretValue::new("default"))) + .await; + match reply { + Ok(None) => assert_eq!(result.unwrap().unwrap().expose(), "default"), + Err(()) => assert!(matches!(result, Err(Error::MissingCiphertext))), + Ok(Some(_)) => unreachable!(), } } diff --git a/litellm-rust/crates/spend-clickhouse/Cargo.toml b/litellm-rust/crates/spend-clickhouse/Cargo.toml index 661a75ac02e..44af53fee6a 100644 --- a/litellm-rust/crates/spend-clickhouse/Cargo.toml +++ b/litellm-rust/crates/spend-clickhouse/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] flate2.workspace = true +litellm-error.workspace = true litellm-http.workspace = true litellm-storage-clickhouse.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/spend-clickhouse/src/error.rs b/litellm-rust/crates/spend-clickhouse/src/error.rs index 0d037ef06f8..c5a9f542ca6 100644 --- a/litellm-rust/crates/spend-clickhouse/src/error.rs +++ b/litellm-rust/crates/spend-clickhouse/src/error.rs @@ -1,15 +1,32 @@ +use litellm_error::Source; + #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("invalid ClickHouse insert row")] - InvalidRow, - #[error("{0} must be a positive integer")] - InvalidLimit(&'static str), - #[error("database must be a nonempty SQL identifier and retention must be positive")] - InvalidSchema, - #[error("ClickHouse insert exceeds the encoded size limit")] - InsertTooLarge, + #[error("{field} must be an integer millisecond timestamp")] + NonIntegerTimestamp { field: &'static str }, + #[error("{field} is outside the supported timestamp range")] + TimestampOutOfRange { field: &'static str, source: Source }, #[error(transparent)] Storage(#[from] litellm_storage_clickhouse::Error), #[error(transparent)] Migration(#[from] sqlx::migrate::MigrateError), } + +impl Error { + pub fn is_request(&self) -> bool { + match self { + Self::NonIntegerTimestamp { .. } | Self::TimestampOutOfRange { .. } => true, + Self::Storage(error) => error.is_request(), + Self::Migration(_) => false, + } + } + + pub fn is_insert_too_large(&self) -> bool { + match self { + Self::Storage(error) => error.is_insert_too_large(), + Self::NonIntegerTimestamp { .. } + | Self::TimestampOutOfRange { .. } + | Self::Migration(_) => false, + } + } +} diff --git a/litellm-rust/crates/spend-clickhouse/src/insert.rs b/litellm-rust/crates/spend-clickhouse/src/insert.rs index 495f0d8ecb2..66d0ea59536 100644 --- a/litellm-rust/crates/spend-clickhouse/src/insert.rs +++ b/litellm-rust/crates/spend-clickhouse/src/insert.rs @@ -13,7 +13,8 @@ use sha2::{Digest, Sha256}; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; use crate::Error; -use litellm_storage_clickhouse::Connection; +use litellm_error::Source; +use litellm_storage_clickhouse::{Connection, Error as StorageError}; fn max_insert_bytes() -> Result { let name = "CLICKHOUSE_TRACE_MAX_INSERT_BYTES"; @@ -22,9 +23,9 @@ fn max_insert_bytes() -> Result { .parse::() .ok() .filter(|value| *value > 0) - .ok_or(Error::InvalidLimit(name)), + .ok_or(StorageError::InvalidLimit(name).into()), Err(std::env::VarError::NotPresent) => Ok(64 * 1024 * 1024), - Err(_) => Err(Error::InvalidLimit(name)), + Err(_) => Err(StorageError::InvalidLimit(name).into()), } } @@ -55,7 +56,11 @@ pub async fn insert_rows( pub fn encode_rows(rows: Vec) -> Result { let body = write_rows(&rows, None, Vec::new(), usize::MAX)?; - String::from_utf8(body).map_err(|_| Error::InvalidRow) + String::from_utf8(body).map_err(invalid_row) +} + +fn invalid_row(error: impl std::error::Error + Send + Sync + 'static) -> Error { + StorageError::InvalidRow(Source::new(error)).into() } fn prepare_insert( @@ -73,9 +78,9 @@ fn prepare_insert( )?; let body = encoder .into_inner() - .map_err(|_| Error::InvalidRow)? + .map_err(|error| invalid_row(error.into_error()))? .finish() - .map_err(|_| Error::InvalidRow)?; + .map_err(invalid_row)?; Ok((token, body)) } @@ -102,7 +107,7 @@ impl Write for LimitedWriter { fn write(&mut self, bytes: &[u8]) -> std::io::Result { if bytes.len() > self.remaining { self.exceeded = true; - return Err(std::io::Error::other(Error::InsertTooLarge)); + return Err(std::io::Error::other(StorageError::InsertTooLarge)); } let written = self.inner.write(bytes)?; self.remaining -= written; @@ -126,17 +131,18 @@ fn write_rows( exceeded: false, }; for (index, row) in rows.iter().enumerate() { + let row = EncodedRow::new(row, received_ms)?; let result = (|| { if index != 0 { writer.write_all(b"\n").map_err(serde_json::Error::io)?; } - serde_json::to_writer(&mut writer, &EncodedRow { row, received_ms }) + serde_json::to_writer(&mut writer, &row) })(); - if result.is_err() { + if let Err(error) = result { return Err(if writer.exceeded { - Error::InsertTooLarge + StorageError::InsertTooLarge.into() } else { - Error::InvalidRow + invalid_row(error) }); } } @@ -144,25 +150,37 @@ fn write_rows( } struct EncodedRow<'a> { - row: &'a InsertRow, + fields: Vec<(&'a str, Cow<'a, Value>)>, received_ms: Option, } +impl<'a> EncodedRow<'a> { + fn new(row: &'a InsertRow, received_ms: Option) -> Result { + let fields = row + .iter() + .map(|(name, value)| Ok((name.as_str(), insert_value(name, value)?))) + .collect::>()?; + Ok(Self { + fields, + received_ms, + }) + } +} + impl Serialize for EncodedRow<'_> { fn serialize(&self, serializer: S) -> Result { let mut map = serializer.serialize_map(None)?; let mut received_ms = self.received_ms; - for (name, value) in self.row { - if name.as_str() >= "EngineReceivedMs" + for (name, value) in &self.fields { + if *name >= "EngineReceivedMs" && let Some(timestamp) = received_ms.take() { map.serialize_entry("EngineReceivedMs", ×tamp)?; } - if name == "EngineReceivedMs" && self.received_ms.is_some() { + if *name == "EngineReceivedMs" && self.received_ms.is_some() { continue; } - let value = insert_value(name, value).map_err(serde::ser::Error::custom)?; - map.serialize_entry(name, &value)?; + map.serialize_entry(name, value)?; } if let Some(timestamp) = received_ms { map.serialize_entry("EngineReceivedMs", ×tamp)?; @@ -171,21 +189,23 @@ impl Serialize for EncodedRow<'_> { } } +const TIMESTAMP_FIELDS: [&str; 3] = ["start_time", "end_time", "completion_start_time"]; + fn insert_value<'a>(name: &str, value: &'a Value) -> Result, Error> { - let multiplier = match name { - "start_time" | "end_time" | "completion_start_time" => 1_000_000, - _ => return Ok(Cow::Borrowed(value)), + let Some(&field) = TIMESTAMP_FIELDS.iter().find(|field| **field == name) else { + return Ok(Cow::Borrowed(value)); }; - if name == "completion_start_time" && value.is_null() { + if field == "completion_start_time" && value.is_null() { return Ok(Cow::Borrowed(value)); } - let timestamp = value.as_i64().ok_or(Error::InvalidRow)?; - let datetime = OffsetDateTime::from_unix_timestamp_nanos(i128::from(timestamp) * multiplier) - .map_err(|_| Error::InvalidRow)?; + let out_of_range = |source| Error::TimestampOutOfRange { field, source }; + let timestamp = value.as_i64().ok_or(Error::NonIntegerTimestamp { field })?; + let datetime = OffsetDateTime::from_unix_timestamp_nanos(i128::from(timestamp) * 1_000_000) + .map_err(|error| out_of_range(Source::new(error)))?; datetime .format(&Rfc3339) .map(|value| Cow::Owned(Value::String(value))) - .map_err(|_| Error::InvalidRow) + .map_err(|error| out_of_range(Source::new(error))) } #[cfg(test)] @@ -204,10 +224,11 @@ mod tests { ]; let encoded = write_rows(&rows, None, Vec::new(), usize::MAX).unwrap(); assert!(write_rows(&rows, None, Vec::new(), encoded.len()).is_ok()); - assert!(matches!( - write_rows(&rows, None, Vec::new(), encoded.len() - 1), - Err(Error::InsertTooLarge) - )); + assert!( + write_rows(&rows, None, Vec::new(), encoded.len() - 1) + .unwrap_err() + .is_insert_too_large() + ); } #[rstest] @@ -264,10 +285,11 @@ mod tests { fn stamped_insert_enforces_the_encoded_limit() { let rows = vec![BTreeMap::new()]; assert!(prepare_insert(&rows, 1, 22).is_ok()); - assert!(matches!( - prepare_insert(&rows, 1, 21), - Err(Error::InsertTooLarge) - )); + assert!( + prepare_insert(&rows, 1, 21) + .unwrap_err() + .is_insert_too_large() + ); } #[rstest] @@ -282,15 +304,36 @@ mod tests { assert_eq!(insert_value(field, &input).unwrap().as_ref(), &expected); } + fn rejected_start_time(input: Value) -> Error { + prepare_insert(&[BTreeMap::from([("start_time".into(), input)])], 456, 4096).unwrap_err() + } + #[rstest] #[case::fractional(json!(1.5))] #[case::string(json!("1234"))] #[case::null(Value::Null)] - #[case::out_of_range(json!(i64::MAX))] - fn invalid_spend_dates_fail_before_transport(#[case] input: Value) { + fn non_integer_spend_dates_fail_before_transport(#[case] input: Value) { + let error = rejected_start_time(input); + assert!(error.is_request()); assert!(matches!( - prepare_insert(&[BTreeMap::from([("start_time".into(), input)])], 456, 4096), - Err(Error::InvalidRow) + error, + Error::NonIntegerTimestamp { + field: "start_time" + } )); } + + #[rstest] + fn out_of_range_spend_dates_keep_their_cause() { + let error = rejected_start_time(json!(i64::MAX)); + assert!(error.is_request()); + let cause = std::error::Error::source(&error) + .and_then(|cause| cause.downcast_ref::()) + .expect("time cause"); + assert!( + cause + .downcast_ref::() + .is_some() + ); + } } diff --git a/litellm-rust/crates/spend-clickhouse/src/lib.rs b/litellm-rust/crates/spend-clickhouse/src/lib.rs index f4c108b1e7f..9da658aa3e8 100644 --- a/litellm-rust/crates/spend-clickhouse/src/lib.rs +++ b/litellm-rust/crates/spend-clickhouse/src/lib.rs @@ -4,7 +4,7 @@ mod schema; pub use error::Error; pub use insert::{encode_rows, insert_rows}; -use litellm_storage_clickhouse::Storage; +use litellm_storage_clickhouse::{Error as StorageError, Storage}; pub use schema::ensure_schema; #[derive(Clone)] @@ -16,7 +16,7 @@ pub struct Config { impl Config { pub fn new(database: String, url: &str, retention_days: u32) -> Result { if retention_days == 0 { - return Err(Error::InvalidSchema); + return Err(StorageError::InvalidSchema.into()); } Ok(Self { storage: Storage::new(database, url)?, diff --git a/litellm-rust/crates/storage-clickhouse/Cargo.toml b/litellm-rust/crates/storage-clickhouse/Cargo.toml index 4e72e823dc2..e768c610098 100644 --- a/litellm-rust/crates/storage-clickhouse/Cargo.toml +++ b/litellm-rust/crates/storage-clickhouse/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] flate2.workspace = true +litellm-error.workspace = true litellm-http.workspace = true serde.workspace = true serde_json.workspace = true @@ -17,6 +18,7 @@ thiserror.workspace = true url.workspace = true [dev-dependencies] +litellm-error.workspace = true litellm-http = { workspace = true, features = ["test-support"] } rstest.workspace = true testcontainers-modules = { version = "0.15.0", features = ["clickhouse"] } diff --git a/litellm-rust/crates/storage-clickhouse/src/error.rs b/litellm-rust/crates/storage-clickhouse/src/error.rs index fa0c5eabd73..ef7b5046455 100644 --- a/litellm-rust/crates/storage-clickhouse/src/error.rs +++ b/litellm-rust/crates/storage-clickhouse/src/error.rs @@ -1,21 +1,23 @@ +use litellm_error::Source; + #[derive(Clone, Debug, thiserror::Error)] pub enum Error { #[error("invalid ClickHouse insert row")] - InvalidRow, + InvalidRow(#[source] Source), #[error("{0} must be a positive integer")] InvalidLimit(&'static str), #[error("invalid ClickHouse insert table")] InvalidTable, #[error("invalid ClickHouse HTTP URL")] - InvalidUrl, + InvalidUrl(#[source] url::ParseError), + #[error("ClickHouse URL must use http or https and name a host")] + UnsupportedUrl, #[error("database must be a nonempty SQL identifier and retention must be positive")] InvalidSchema, #[error("SQL query must not be empty")] EmptySql, #[error("invalid ClickHouse query parameters")] - InvalidParameters, - #[error("unknown ClickHouse read query")] - InvalidQuery, + InvalidParameters(#[source] Source), #[error("ClickHouse query failed with HTTP status {0}")] QueryFailed(u16), #[error("ClickHouse insert failed with HTTP status {0}")] @@ -24,10 +26,60 @@ pub enum Error { InsertTooLarge, #[error("ClickHouse schema setup failed with HTTP status {0}")] SchemaFailed(u16), + #[error("ClickHouse migrations are forward-only")] + Irreversible, #[error("ClickHouse query exceeded the response size limit")] ResponseTooLarge, - #[error("ClickHouse returned an invalid or failed JSON query response")] - InvalidResponse, - #[error("ClickHouse query transport failed")] - Transport, + #[error("ClickHouse returned an invalid response")] + InvalidResponse(#[source] Source), + #[error("ClickHouse returned a failed or unexpected response")] + UnexpectedResponse, + #[error("ClickHouse transport failed")] + Transport(#[from] litellm_http::Error), +} + +impl Error { + pub fn is_request(&self) -> bool { + match self { + Self::InvalidRow(_) + | Self::InvalidLimit(_) + | Self::InvalidTable + | Self::InvalidSchema + | Self::EmptySql + | Self::InvalidParameters(_) => true, + Self::InvalidUrl(_) + | Self::UnsupportedUrl + | Self::QueryFailed(_) + | Self::InsertFailed(_) + | Self::InsertTooLarge + | Self::SchemaFailed(_) + | Self::Irreversible + | Self::ResponseTooLarge + | Self::InvalidResponse(_) + | Self::UnexpectedResponse + | Self::Transport(_) => false, + } + } + + pub fn is_insert_too_large(&self) -> bool { + match self { + Self::InsertTooLarge => true, + Self::InvalidRow(_) + | Self::InvalidLimit(_) + | Self::InvalidTable + | Self::InvalidUrl(_) + | Self::UnsupportedUrl + | Self::InvalidSchema + | Self::EmptySql + | Self::InvalidParameters(_) + | Self::QueryFailed(_) + | Self::InsertFailed(_) + | Self::SchemaFailed(_) + | Self::Irreversible + | Self::ResponseTooLarge + | Self::InvalidResponse(_) + | Self::UnexpectedResponse + | Self::Transport(_) => false, + } + } } diff --git a/litellm-rust/crates/storage-clickhouse/src/insert.rs b/litellm-rust/crates/storage-clickhouse/src/insert.rs index 1cdad1897c1..29837359f0b 100644 --- a/litellm-rust/crates/storage-clickhouse/src/insert.rs +++ b/litellm-rust/crates/storage-clickhouse/src/insert.rs @@ -1,6 +1,7 @@ use std::{io::Write, time::Duration}; use flate2::{Compression, write::GzEncoder}; +use litellm_error::Source; use litellm_http::Client; use crate::{Connection, Error, valid_identifier}; @@ -34,13 +35,15 @@ pub async fn insert_encoded_rows( return Err(Error::InvalidTable); } let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - encoder - .write_all(encoded.as_bytes()) - .map_err(|_| Error::InvalidRow)?; - let body = encoder.finish().map_err(|_| Error::InvalidRow)?; + encoder.write_all(encoded.as_bytes()).map_err(invalid_row)?; + let body = encoder.finish().map_err(invalid_row)?; insert_compressed_rows(client, connection, database, table, token, body).await } +fn invalid_row(error: std::io::Error) -> Error { + Error::InvalidRow(Source::new(error)) +} + pub async fn insert_compressed_rows( client: &Client, connection: &Connection, @@ -91,7 +94,7 @@ pub async fn insert_compressed_rows( .body(body) .send() .await - .map_err(|_| Error::Transport)?; + .map_err(litellm_http::Error::from)?; if !response.status().is_success() { return Err(Error::InsertFailed(response.status().as_u16())); } diff --git a/litellm-rust/crates/storage-clickhouse/src/lib.rs b/litellm-rust/crates/storage-clickhouse/src/lib.rs index e2ef62d1bc3..60a4e77a36c 100644 --- a/litellm-rust/crates/storage-clickhouse/src/lib.rs +++ b/litellm-rust/crates/storage-clickhouse/src/lib.rs @@ -16,9 +16,9 @@ pub struct Connection { impl Connection { pub fn parse(value: &str) -> Result { - let url = Url::parse(value).map_err(|_| Error::InvalidUrl)?; + let url = Url::parse(value).map_err(Error::InvalidUrl)?; if !matches!(url.scheme(), "http" | "https") || url.host().is_none() { - return Err(Error::InvalidUrl); + return Err(Error::UnsupportedUrl); } Ok(Self { url }) } @@ -33,11 +33,11 @@ impl Connection { connection .url .set_username(user) - .map_err(|_| Error::InvalidUrl)?; + .map_err(|()| Error::UnsupportedUrl)?; connection .url .set_password(Some(password)) - .map_err(|_| Error::InvalidUrl)?; + .map_err(|()| Error::UnsupportedUrl)?; let pairs: Vec<_> = connection .url .query_pairs() diff --git a/litellm-rust/crates/storage-clickhouse/src/migrate.rs b/litellm-rust/crates/storage-clickhouse/src/migrate.rs index c249020faaf..4da26baa681 100644 --- a/litellm-rust/crates/storage-clickhouse/src/migrate.rs +++ b/litellm-rust/crates/storage-clickhouse/src/migrate.rs @@ -12,7 +12,7 @@ use sqlx::{ migrate::{AppliedMigration, Migrate, MigrateError, Migration}, }; -use crate::{Connection, Error, READ_LIMITS, valid_identifier}; +use crate::{Connection, Error, READ_LIMITS, read::invalid_response, valid_identifier}; pub async fn execute_statement( client: &Client, @@ -22,7 +22,7 @@ pub async fn execute_statement( ) -> Result<(), Error> { let body = execute_sql(client, connection, sql, timeout).await?; if !body.trim().is_empty() { - return Err(Error::InvalidResponse); + return Err(Error::UnexpectedResponse); } Ok(()) } @@ -56,18 +56,18 @@ async fn execute_sql( .body(sql.to_owned()) .send() .await - .map_err(|_| Error::Transport)?; + .map_err(litellm_http::Error::from)?; if !response.status().is_success() { return Err(Error::SchemaFailed(response.status().as_u16())); } let mut body = Vec::new(); - while let Some(chunk) = response.chunk().await.map_err(|_| Error::Transport)? { + while let Some(chunk) = response.chunk().await.map_err(litellm_http::Error::from)? { if body.len() + chunk.len() > READ_LIMITS.response_bytes { return Err(Error::ResponseTooLarge); } body.extend_from_slice(&chunk); } - String::from_utf8(body).map_err(|_| Error::InvalidResponse) + String::from_utf8(body).map_err(invalid_response) } /// The startup runner records success after execution, never marks migrations dirty, and skips locks @@ -131,13 +131,13 @@ fn encode_hex(bytes: &[u8]) -> String { fn decode_hex(value: &str) -> Result, Error> { let (pairs, remainder) = value.as_bytes().as_chunks::<2>(); if !remainder.is_empty() { - return Err(Error::InvalidResponse); + return Err(Error::UnexpectedResponse); } pairs .iter() .map(|pair| { - let high = decode_hex_digit(pair[0]).ok_or(Error::InvalidResponse)?; - let low = decode_hex_digit(pair[1]).ok_or(Error::InvalidResponse)?; + let high = decode_hex_digit(pair[0]).ok_or(Error::UnexpectedResponse)?; + let low = decode_hex_digit(pair[1]).ok_or(Error::UnexpectedResponse)?; Ok((high << 4) | low) }) .collect() @@ -231,7 +231,7 @@ where .filter(|line| !line.trim().is_empty()) .map(|line| { let row = serde_json::from_str::(line) - .map_err(|_| migrate_error(Error::InvalidResponse))?; + .map_err(|error| migrate_error(invalid_response(error)))?; let checksum = decode_hex(&row.checksum).map_err(migrate_error)?; Ok(AppliedMigration { version: row.version, @@ -288,11 +288,7 @@ where _table_name: &'e str, _migration: &'e Migration, ) -> MigrateFuture<'e, Result> { - Box::pin(async { - Err(MigrateError::Execute(SqlxError::AnyDriverError(Box::new( - std::io::Error::other("ClickHouse migrations are forward-only"), - )))) - }) + Box::pin(async { Err(migrate_error(Error::Irreversible)) }) } } diff --git a/litellm-rust/crates/storage-clickhouse/src/read.rs b/litellm-rust/crates/storage-clickhouse/src/read.rs index 03a5522e595..40d4ad0a073 100644 --- a/litellm-rust/crates/storage-clickhouse/src/read.rs +++ b/litellm-rust/crates/storage-clickhouse/src/read.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, time::Duration}; +use litellm_error::Source; use litellm_http::Client; use serde::{Deserialize, Serialize, de::DeserializeOwned}; @@ -115,7 +116,7 @@ async fn execute_read_with_limits( .post(url) .timeout(Duration::from_secs(15)) .body(sql.to_owned()); - let mut response = request.send().await.map_err(|_| Error::Transport)?; + let mut response = request.send().await.map_err(litellm_http::Error::from)?; if !response.status().is_success() { if response .headers() @@ -128,20 +129,19 @@ async fn execute_read_with_limits( } let mut body = Vec::new(); - while let Some(chunk) = response.chunk().await.map_err(|_| Error::Transport)? { + while let Some(chunk) = response.chunk().await.map_err(litellm_http::Error::from)? { if body.len() + chunk.len() > limits.response_bytes { return Err(Error::ResponseTooLarge); } body.extend_from_slice(&chunk); } - let json: serde_json::Value = - serde_json::from_slice(&body).map_err(|_| Error::InvalidResponse)?; + let json: serde_json::Value = serde_json::from_slice(&body).map_err(invalid_response)?; if json.get("exception").is_some() || !json.get("data").is_some_and(serde_json::Value::is_array) { - return Err(Error::InvalidResponse); + return Err(Error::UnexpectedResponse); } - String::from_utf8(body).map_err(|_| Error::InvalidResponse) + String::from_utf8(body).map_err(invalid_response) } pub trait Query { @@ -158,8 +158,8 @@ struct Rows { } fn parameters(params: &T) -> Result, Error> { - let value = serde_json::to_value(params).map_err(|_| Error::InvalidParameters)?; - serde_json::from_value(value).map_err(|_| Error::InvalidParameters) + let value = serde_json::to_value(params).map_err(invalid_parameters)?; + serde_json::from_value(value).map_err(invalid_parameters) } pub async fn fetch( @@ -198,5 +198,13 @@ pub async fn fetch_json( fn decode_rows(body: &str) -> Result, Error> { serde_json::from_str::>(body) .map(|rows| rows.data) - .map_err(|_| Error::InvalidResponse) + .map_err(invalid_response) +} + +pub(crate) fn invalid_response(error: impl std::error::Error + Send + Sync + 'static) -> Error { + Error::InvalidResponse(Source::new(error)) +} + +fn invalid_parameters(error: serde_json::Error) -> Error { + Error::InvalidParameters(Source::new(error)) } diff --git a/litellm-rust/crates/storage-clickhouse/tests/migrations.rs b/litellm-rust/crates/storage-clickhouse/tests/migrations.rs index 49f481df632..09e56766891 100644 --- a/litellm-rust/crates/storage-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/storage-clickhouse/tests/migrations.rs @@ -346,11 +346,7 @@ async fn revert_reports_forward_only_error() { .revert("_sqlx_migrations", &migration) .await .expect_err("ClickHouse migrations cannot be reverted"); - assert!( - error - .to_string() - .contains("ClickHouse migrations are forward-only") - ); + assert!(matches!(storage_error(&error), Some(Error::Irreversible))); } #[rstest] @@ -391,6 +387,7 @@ async fn applied_int64_versions_accept_numeric_and_quoted_json( #[tokio::test] async fn schema_requests_override_unsafe_connection_settings( #[future(awt)] mock_server: MockServer, + #[values("wait_end_of_query", "send_progress_in_http_headers", "async_insert")] name: &str, ) { Mock::given(method("POST")) .and(query_param("wait_end_of_query", "1")) @@ -417,20 +414,14 @@ async fn schema_requests_override_unsafe_connection_settings( .received_requests() .await .expect("requests recorded"); - for name in [ - "wait_end_of_query", - "send_progress_in_http_headers", - "async_insert", - ] { - assert_eq!( - requests[0] - .url - .query_pairs() - .filter(|(key, _)| key == name) - .count(), - 1 - ); - } + assert_eq!( + requests[0] + .url + .query_pairs() + .filter(|(key, _)| key == name) + .count(), + 1 + ); } #[rstest] diff --git a/litellm-rust/crates/storage-clickhouse/tests/transport.rs b/litellm-rust/crates/storage-clickhouse/tests/transport.rs index 2560c2985f0..6b2c3fb8ef1 100644 --- a/litellm-rust/crates/storage-clickhouse/tests/transport.rs +++ b/litellm-rust/crates/storage-clickhouse/tests/transport.rs @@ -55,18 +55,24 @@ impl litellm_storage_clickhouse::Query for TypedQuery { const SQL: &'static str = "SELECT typed_parameters"; } +enum Expected { + Rows, + Undecodable, + Failed, +} + #[rstest] #[case::valid( r#"{"meta":[],"data":[{"answer":"ok"}],"rows":1,"statistics":{"elapsed":0.1}}"#, - true + Expected::Rows )] -#[case::wrong_type(r#"{"data":[{"answer":1}]}"#, false)] -#[case::missing_column(r#"{"data":[{}]}"#, false)] -#[case::exception(r#"{"data":[],"exception":"failed"}"#, false)] +#[case::wrong_type(r#"{"data":[{"answer":1}]}"#, Expected::Undecodable)] +#[case::missing_column(r#"{"data":[{}]}"#, Expected::Undecodable)] +#[case::exception(r#"{"data":[],"exception":"failed"}"#, Expected::Failed)] #[tokio::test] async fn typed_fetch_encodes_parameters_and_validates_rows( #[case] body: &str, - #[case] valid: bool, + #[case] expected: Expected, ) { use litellm_storage_clickhouse::{fetch, fetch_json}; use wiremock::{ @@ -98,17 +104,24 @@ async fn typed_fetch_encodes_parameters_and_validates_rows( }; let rows = fetch::(&client, &connection, ¶ms).await; let envelope = fetch_json::(&client, &connection, ¶ms).await; - if valid { - assert_eq!( - rows.unwrap(), - vec![QueryRow { - answer: "ok".into() - }] - ); - assert_eq!(envelope.unwrap(), body); - } else { - assert!(matches!(rows, Err(Error::InvalidResponse))); - assert!(matches!(envelope, Err(Error::InvalidResponse))); + match expected { + Expected::Rows => { + assert_eq!( + rows.unwrap(), + vec![QueryRow { + answer: "ok".into() + }] + ); + assert_eq!(envelope.unwrap(), body); + } + Expected::Undecodable => { + assert!(matches!(rows, Err(Error::InvalidResponse(_)))); + assert!(matches!(envelope, Err(Error::InvalidResponse(_)))); + } + Expected::Failed => { + assert!(matches!(rows, Err(Error::UnexpectedResponse))); + assert!(matches!(envelope, Err(Error::UnexpectedResponse))); + } } } @@ -145,21 +158,23 @@ async fn server_result_limits_allow_smaller_pages_without_retrying_other_failure } } -#[test] -fn insert_timeout_environment_controls_transport() { - for value in ["1", "3", "0", "invalid"] { - let result = std::process::Command::new(std::env::current_exe().unwrap()) - .args(["--exact", "insert_timeout_environment_child"]) - .env("LITELLM_TEST_INSERT_TIMEOUT", value) - .env("CLICKHOUSE_INSERT_TIMEOUT_SECONDS", value) - .output() - .unwrap(); - assert!( - result.status.success(), - "{}", - String::from_utf8_lossy(&result.stdout) - ); - } +#[rstest] +#[case::times_out("1")] +#[case::completes("3")] +#[case::zero("0")] +#[case::not_a_number("invalid")] +fn insert_timeout_environment_controls_transport(#[case] value: &str) { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "insert_timeout_environment_child"]) + .env("LITELLM_TEST_INSERT_TIMEOUT", value) + .env("CLICKHOUSE_INSERT_TIMEOUT_SECONDS", value) + .output() + .unwrap(); + assert!( + result.status.success(), + "{}", + String::from_utf8_lossy(&result.stdout) + ); } #[tokio::test] @@ -183,7 +198,7 @@ async fn insert_timeout_environment_child() { ) .await; match value.as_str() { - "1" => assert!(matches!(result, Err(Error::Transport))), + "1" => assert!(matches!(result, Err(Error::Transport(error)) if error.dispatched())), "3" => assert!(result.is_ok()), _ => assert!(matches!( result, @@ -191,3 +206,29 @@ async fn insert_timeout_environment_child() { )), } } + +#[rstest] +#[tokio::test] +async fn unreachable_server_keeps_the_transport_cause() { + let address = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap(); + let connection = Connection::parse(&format!("http://{address}")).unwrap(); + let error = execute_read( + &Client::for_test(), + &connection, + "SELECT 1", + &BTreeMap::new(), + ) + .await + .unwrap_err(); + let cause = std::error::Error::source(&error) + .and_then(|cause| cause.downcast_ref::()) + .expect("transport cause"); + assert!(!cause.dispatched()); + assert_ne!( + litellm_error::Report::new(&error).to_string(), + error.to_string() + ); +} diff --git a/litellm-rust/crates/testkit/Cargo.toml b/litellm-rust/crates/testkit/Cargo.toml index 97efb753384..ef3c3e6cd7e 100644 --- a/litellm-rust/crates/testkit/Cargo.toml +++ b/litellm-rust/crates/testkit/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] flate2.workspace = true +litellm-http.workspace = true reqwest.workspace = true serde.workspace = true semver.workspace = true diff --git a/litellm-rust/crates/testkit/src/agent/configure.rs b/litellm-rust/crates/testkit/src/agent/configure.rs index a41bec9379a..6f1b970e8fb 100644 --- a/litellm-rust/crates/testkit/src/agent/configure.rs +++ b/litellm-rust/crates/testkit/src/agent/configure.rs @@ -5,10 +5,13 @@ use semver::Version; use crate::Error; -#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::VariantArray)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::VariantArray, strum::Display)] pub enum Wire { + #[strum(serialize = "chat completions")] ChatCompletions, + #[strum(serialize = "messages")] Messages, + #[strum(serialize = "responses")] Responses, } diff --git a/litellm-rust/crates/testkit/src/error.rs b/litellm-rust/crates/testkit/src/error.rs index 03520d827f1..8b747c28bda 100644 --- a/litellm-rust/crates/testkit/src/error.rs +++ b/litellm-rust/crates/testkit/src/error.rs @@ -15,7 +15,7 @@ pub enum Error { Request { url: String, #[source] - source: reqwest::Error, + source: litellm_http::Error, }, #[error("{url} answered with status {status}")] Status { url: String, status: u16 }, @@ -47,10 +47,14 @@ pub enum Error { expected: String, reported: String, }, - #[error("{agent} cannot talk to the gateway over {wire:?}")] + #[error("{agent} cannot talk to the gateway over {wire}")] UnsupportedWire { agent: &'static str, wire: Wire }, - #[error("agent did not finish within {0:?}")] + #[error("agent did not finish within {} seconds", .0.as_secs_f64())] Timeout(std::time::Duration), - #[error("io failure")] - Io(#[from] io::Error), + #[error("could not install the binary into {}", dir.display())] + Install { dir: PathBuf, source: io::Error }, + #[error("could not run {}", binary.display())] + Spawn { binary: PathBuf, source: io::Error }, + #[error("could not write agent files under {}", home.display())] + WriteFiles { home: PathBuf, source: io::Error }, } diff --git a/litellm-rust/crates/testkit/src/install/fetch.rs b/litellm-rust/crates/testkit/src/install/fetch.rs index 73008f7a0da..59a61bfaa0a 100644 --- a/litellm-rust/crates/testkit/src/install/fetch.rs +++ b/litellm-rust/crates/testkit/src/install/fetch.rs @@ -38,9 +38,9 @@ impl Fetch for HttpFetch { (Some(token), true) => request.bearer_auth(token), _ => request, }; - let request_error = |source| Error::Request { + let request_error = |source: reqwest::Error| Error::Request { url: url.to_owned(), - source, + source: source.into(), }; let response = request.send().await.map_err(request_error)?; let status = response.status(); diff --git a/litellm-rust/crates/testkit/src/install/mod.rs b/litellm-rust/crates/testkit/src/install/mod.rs index 1104bcec102..73f7deec864 100644 --- a/litellm-rust/crates/testkit/src/install/mod.rs +++ b/litellm-rust/crates/testkit/src/install/mod.rs @@ -52,7 +52,13 @@ impl Installer { version: version.clone(), binary: binary.clone(), }; - if fs::try_exists(&binary).await? && probe_version(&binary, version).await.is_ok() { + let install_error = |source| Error::Install { + dir: dir.clone(), + source, + }; + if fs::try_exists(&binary).await.map_err(install_error)? + && probe_version(&binary, version).await.is_ok() + { return Ok(installed); } @@ -61,21 +67,23 @@ impl Installer { verify_sha256(&release.asset, &release.sha256, &archive)?; let contents = extract_binary(&release.packaging, &archive)?; - fs::create_dir_all(&dir).await?; + fs::create_dir_all(&dir).await.map_err(install_error)?; let staging = dir.join(format!( ".{}.{}.{}.partial", agent.binary(), std::process::id(), STAGING_COUNTER.fetch_add(1, Ordering::Relaxed) )); - fs::write(&staging, contents).await?; - fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o755)).await?; - fs::rename(&staging, &binary).await?; + fs::write(&staging, contents).await.map_err(install_error)?; + fs::set_permissions(&staging, std::fs::Permissions::from_mode(0o755)) + .await + .map_err(install_error)?; + fs::rename(&staging, &binary).await.map_err(install_error)?; match probe_version(&binary, version).await { Ok(()) => Ok(installed), Err(error) => { - fs::remove_file(&binary).await?; + fs::remove_file(&binary).await.map_err(install_error)?; Err(error) } } @@ -98,7 +106,11 @@ async fn probe_version(binary: &Path, expected: &Version) -> Result<(), Error> { .env("DISABLE_AUTOUPDATER", "1") .stdin(Stdio::null()) .output() - .await?; + .await + .map_err(|source| Error::Spawn { + binary: binary.to_owned(), + source, + })?; let stdout = String::from_utf8_lossy(&output.stdout); if stdout .split_whitespace() diff --git a/litellm-rust/crates/testkit/src/session.rs b/litellm-rust/crates/testkit/src/session.rs index 6b06e756cca..cd5eb08ae2b 100644 --- a/litellm-rust/crates/testkit/src/session.rs +++ b/litellm-rust/crates/testkit/src/session.rs @@ -28,7 +28,11 @@ impl Session { ) -> Result { let home = home.into(); let spec = agent.configure(&installed.version, &settings, &home)?; - spec.write_files(&home)?; + spec.write_files(&home) + .map_err(|source| Error::WriteFiles { + home: home.clone(), + source, + })?; Ok(Self { binary: installed.binary.clone(), home, @@ -55,7 +59,11 @@ impl Session { .output(); let output = timeout(limit, child) .await - .map_err(|_| Error::Timeout(limit))??; + .map_err(|_| Error::Timeout(limit))? + .map_err(|source| Error::Spawn { + binary: self.binary.clone(), + source, + })?; let parsed = agent.parse(&self.version, &String::from_utf8_lossy(&output.stdout)); let failed_silently = !output.status.success() && parsed.errors.is_empty(); Ok(Outcome { diff --git a/litellm-rust/crates/token-counter-fast/Cargo.toml b/litellm-rust/crates/token-counter-fast/Cargo.toml index 5127dc17f22..070bec58519 100644 --- a/litellm-rust/crates/token-counter-fast/Cargo.toml +++ b/litellm-rust/crates/token-counter-fast/Cargo.toml @@ -13,6 +13,7 @@ tokenizers.workspace = true unicode-normalization-alignments = "0.1.12" [dev-dependencies] +litellm-error.workspace = true rand.workspace = true rstest.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/token-counter-fast/src/error.rs b/litellm-rust/crates/token-counter-fast/src/error.rs index d2a706191ac..2cd8d5c810c 100644 --- a/litellm-rust/crates/token-counter-fast/src/error.rs +++ b/litellm-rust/crates/token-counter-fast/src/error.rs @@ -2,16 +2,25 @@ use thiserror::Error as ThisError; #[derive(Debug, ThisError)] pub enum Error { - #[error("failed to load tokenizer: {0}")] + #[error("failed to load tokenizer")] Load(#[source] tokenizers::Error), - #[error("failed to load tokenizer: tiktoken rank file: {0}")] + #[error("failed to load tokenizer from a tiktoken rank file")] Ranks(#[source] RankError), #[error("failed to load tokenizer: Unicode character classes are unavailable")] UnicodeClasses, - #[error("tokenization failed: {0}")] + #[error("tokenization failed")] Encode(#[source] tokenizers::Error), } +impl Error { + pub fn is_load(&self) -> bool { + match self { + Self::Load(_) | Self::Ranks(_) | Self::UnicodeClasses => true, + Self::Encode(_) => false, + } + } +} + #[derive(Debug, ThisError)] pub enum RankError { #[error("rank {0} is reserved")] @@ -20,8 +29,8 @@ pub enum RankError { MissingByte(u8), #[error("line without a rank: {0:?}")] MissingRank(String), - #[error("token is not base64: {0}")] + #[error("token is not base64")] InvalidToken(#[source] base64::DecodeError), - #[error("rank is not an integer: {0}")] + #[error("rank is not an integer")] InvalidRank(#[source] std::num::ParseIntError), } diff --git a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs index f2d4de38f8c..11182e84e8d 100644 --- a/litellm-rust/crates/token-counter-fast/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter-fast/src/tiktoken.rs @@ -254,8 +254,12 @@ mod tests { .err() .expect("invalid rank file"); assert!(matches!(error, Error::Ranks(_))); - assert!(error.to_string().starts_with(&format!( - "failed to load tokenizer: tiktoken rank file: {detail}" - ))); + assert!( + litellm_error::Report::new(&error) + .to_string() + .starts_with(&format!( + "failed to load tokenizer from a tiktoken rank file: {detail}" + )) + ); } } diff --git a/litellm-rust/crates/token-counter-huggingface/src/error.rs b/litellm-rust/crates/token-counter-huggingface/src/error.rs index e7f6260321b..b4e09db253f 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/error.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/error.rs @@ -2,10 +2,21 @@ use thiserror::Error as ThisError; #[derive(Debug, ThisError)] pub enum Error { - #[error("failed to load tokenizer: {0}")] + #[error("failed to load tokenizer")] Load(#[source] tokenizers::Error), - #[error("tokenization failed: {0}")] + #[error("invalid encoding JSON")] + EncodingJson(#[source] serde_json::Error), + #[error("tokenization failed")] Encode(#[source] tokenizers::Error), - #[error("token decoding failed: {0}")] + #[error("token decoding failed")] Decode(#[source] tokenizers::Error), } + +impl Error { + pub fn is_load(&self) -> bool { + match self { + Self::Load(_) | Self::EncodingJson(_) => true, + Self::Encode(_) | Self::Decode(_) => false, + } + } +} diff --git a/litellm-rust/crates/token-counter-huggingface/src/lib.rs b/litellm-rust/crates/token-counter-huggingface/src/lib.rs index 0acea0532f8..c3af9b42767 100644 --- a/litellm-rust/crates/token-counter-huggingface/src/lib.rs +++ b/litellm-rust/crates/token-counter-huggingface/src/lib.rs @@ -13,11 +13,11 @@ pub use tokenizers::{ }; pub fn encoding_from_json(json: &str) -> Result { - serde_json::from_str(json).map_err(|error| Error::Load(error.into())) + serde_json::from_str(json).map_err(Error::EncodingJson) } pub fn encoding_to_json(encoding: &Encoding) -> Result { - serde_json::to_string(encoding).map_err(|error| Error::Load(error.into())) + serde_json::to_string(encoding).map_err(Error::EncodingJson) } pub struct HuggingFaceTokenizer { diff --git a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml index d1789a693a3..bca7345dcd4 100644 --- a/litellm-rust/crates/token-counter-tiktoken/Cargo.toml +++ b/litellm-rust/crates/token-counter-tiktoken/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true [dependencies] base64.workspace = true +litellm-error.workspace = true once_cell = "1.21.3" rustc-hash = "2.1.3" thiserror.workspace = true diff --git a/litellm-rust/crates/token-counter-tiktoken/src/error.rs b/litellm-rust/crates/token-counter-tiktoken/src/error.rs index e28cbbfb620..024be00f048 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/error.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/error.rs @@ -1,5 +1,50 @@ +use std::{io, num::ParseIntError}; + +use litellm_error::Source; use thiserror::Error as ThisError; +use tiktoken_rs::{DecodeKeyError, EncodeError}; #[derive(Debug, ThisError)] -#[error("unsupported tokenizer: {0}")] -pub struct UnsupportedTokenizer(pub String); +pub enum Error { + #[error("unsupported tokenizer: {0}")] + UnsupportedTokenizer(String), + #[error("could not read tiktoken rank file {file}")] + Read { + file: &'static str, + source: io::Error, + }, + #[error("invalid tiktoken rank file")] + Ranks(#[from] RankError), + #[error("could not build the tiktoken encoder")] + Encoder(#[source] Source), + #[error("tokenization failed")] + Encode(#[source] EncodeError), + #[error("token decoding failed")] + Decode(#[source] DecodeKeyError), +} + +impl Error { + pub fn is_load(&self) -> bool { + match self { + Self::UnsupportedTokenizer(_) + | Self::Read { .. } + | Self::Ranks(_) + | Self::Encoder(_) => true, + Self::Encode(_) | Self::Decode(_) => false, + } + } +} + +#[derive(Debug, ThisError)] +pub enum RankError { + #[error("line without a rank: {0:?}")] + MissingRank(String), + #[error("token is not base64")] + InvalidToken(#[source] base64::DecodeError), + #[error("rank is not an integer")] + InvalidRank(#[source] ParseIntError), + #[error("a rank is assigned to more than one token")] + DuplicateRank, + #[error("byte 0x{0:02X} has no token")] + MissingByte(u8), +} diff --git a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs index 30d0f820da3..c4cb786d9e4 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/lib.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/lib.rs @@ -5,8 +5,8 @@ mod ranks; use std::collections::HashSet; -pub use error::UnsupportedTokenizer; -pub use ranks::{LoadError, Vocabulary}; +pub use error::{Error, RankError}; +pub use ranks::Vocabulary; pub struct TiktokenTokenizer { encoder: &'static tiktoken_rs::CoreBPE, @@ -22,7 +22,7 @@ impl TiktokenTokenizer { pub fn from_cached_ranks( name: &str, load: impl FnOnce(&str) -> std::io::Result, - ) -> Result { + ) -> Result { let (loaded, name) = ranks::load(name, load)?; Ok(Self { encoder: &loaded.bpe, @@ -32,7 +32,7 @@ impl TiktokenTokenizer { } /// The encodings tiktoken-rs embeds, for hosts without the packaged rank files. - pub fn from_name(name: &str) -> Result { + pub fn from_name(name: &str) -> Result { let (encoder, name) = match name { "cl100k_base" => (tiktoken_rs::cl100k_base_singleton(), "cl100k_base"), "o200k_base" => (tiktoken_rs::o200k_base_singleton(), "o200k_base"), @@ -41,7 +41,7 @@ impl TiktokenTokenizer { "p50k_edit" => (tiktoken_rs::p50k_edit_singleton(), "p50k_edit"), "r50k_base" => (tiktoken_rs::r50k_base_singleton(), "r50k_base"), "gpt2" => (tiktoken_rs::r50k_base_singleton(), "gpt2"), - _ => return Err(UnsupportedTokenizer(name.to_owned())), + _ => return Err(Error::UnsupportedTokenizer(name.to_owned())), }; Ok(Self { encoder, @@ -62,12 +62,12 @@ impl TiktokenTokenizer { self.encoder.encode_ordinary(text) } - pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result, String> { + pub fn encode_special(&self, text: &str, allowed: &[String]) -> Result, Error> { let allowed = allowed.iter().map(String::as_str).collect(); self.encoder .encode(text, &allowed) .map(|(ids, _)| ids) - .map_err(|error| error.to_string()) + .map_err(Error::Encode) } pub fn special_tokens(&self) -> HashSet { @@ -92,17 +92,13 @@ impl TiktokenTokenizer { (stable, completions) } - pub fn decode_bytes(&self, ids: &[u32]) -> Result, String> { - self.encoder - .decode_bytes(ids) - .map_err(|error| error.to_string()) + pub fn decode_bytes(&self, ids: &[u32]) -> Result, Error> { + self.encoder.decode_bytes(ids).map_err(Error::Decode) } - pub fn decode(&self, ids: &[u32]) -> Result { - self.encoder - .decode_bytes(ids) + pub fn decode(&self, ids: &[u32]) -> Result { + self.decode_bytes(ids) .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) - .map_err(|error| error.to_string()) } pub fn name(&self) -> &str { @@ -161,7 +157,8 @@ mod tests { #[test] fn unsupported_encoding_preserves_its_name() { - let Err(UnsupportedTokenizer(name)) = TiktokenTokenizer::from_name("unknown-encoding") + let Err(Error::UnsupportedTokenizer(name)) = + TiktokenTokenizer::from_name("unknown-encoding") else { panic!("unknown encoding must be rejected"); }; diff --git a/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs index 192e6f65a0b..3a9cf7d2d3f 100644 --- a/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs +++ b/litellm-rust/crates/token-counter-tiktoken/src/ranks.rs @@ -1,10 +1,10 @@ use base64::{Engine, engine::general_purpose::STANDARD}; +use litellm_error::Source; use once_cell::sync::OnceCell; use rustc_hash::FxHashMap; -use thiserror::Error; use tiktoken_rs::{CoreBPE, O200K_BASE_PAT_STR, Rank}; -use crate::UnsupportedTokenizer; +use crate::error::{Error, RankError}; const CL100K: &str = "9b5ad71b2ce5302211f9c61530b329a4922fc6a4"; const O200K: &str = "fb374d419588a4632f3f557e76b4b70aebbca790"; @@ -76,21 +76,13 @@ impl Vocabulary { } } -#[derive(Debug, Error)] -pub enum LoadError { - #[error(transparent)] - Unsupported(#[from] UnsupportedTokenizer), - #[error("failed to load tiktoken ranks: {0}")] - Ranks(String), -} - /// Loads `name` once per process. The returned name is the one requested (`gpt2` stays /// `gpt2`, as `tiktoken.get_encoding("gpt2").name` does), while `gpt2` and `r50k_base` share /// one cached encoder. pub(super) fn load( name: &str, load_file: impl FnOnce(&str) -> std::io::Result, -) -> Result<(&'static Loaded, &'static str), LoadError> { +) -> Result<(&'static Loaded, &'static str), Error> { let (requested, canonical, file, cache) = match name { "cl100k_base" => ("cl100k_base", "cl100k_base", CL100K, &CL100K_ENCODER), "o200k_base" => ("o200k_base", "o200k_base", O200K, &O200K_ENCODER), @@ -99,16 +91,16 @@ pub(super) fn load( "p50k_edit" => ("p50k_edit", "p50k_edit", P50K, &EDIT_ENCODER), "r50k_base" => ("r50k_base", "r50k_base", P50K, &R50K_ENCODER), "gpt2" => ("gpt2", "r50k_base", P50K, &R50K_ENCODER), - _ => return Err(UnsupportedTokenizer(name.to_owned()).into()), + _ => return Err(Error::UnsupportedTokenizer(name.to_owned())), }; let loaded = cache.get_or_try_init(|| { - let ranks = load_file(file).map_err(|error| LoadError::Ranks(error.to_string()))?; + let ranks = load_file(file).map_err(|source| Error::Read { file, source })?; build(canonical, &ranks) })?; Ok((loaded, requested)) } -fn build(name: &str, ranks: &str) -> Result { +fn build(name: &str, ranks: &str) -> Result { let parsed = ranks .lines() .map(parse_rank) @@ -122,9 +114,11 @@ fn build(name: &str, ranks: &str) -> Result { .collect::>() .len() != encoder.len() - || (0..=u8::MAX).any(|byte| !encoder.contains_key(&[byte][..])) { - return Err(LoadError::Ranks("invalid vocabulary ranks".into())); + return Err(RankError::DuplicateRank.into()); + } + if let Some(byte) = (0..=u8::MAX).find(|byte| !encoder.contains_key(&[*byte][..])) { + return Err(RankError::MissingByte(byte).into()); } let (pattern, specials): (&str, &[(&str, Rank)]) = match name { "cl100k_base" => ( @@ -185,9 +179,12 @@ fn build(name: &str, ranks: &str) -> Result { .chain(special_tokens.values()) .copied() .max() - .ok_or_else(|| LoadError::Ranks("empty vocabulary".into()))?; - let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern) - .map_err(|error| LoadError::Ranks(error.to_string()))?; + .unwrap_or_default(); + let bpe = CoreBPE::new(encoder.clone(), special_tokens.clone(), pattern).map_err(|error| { + Error::Encoder(Source::from( + Box::::from(error), + )) + })?; Ok(Loaded { bpe, vocabulary: Vocabulary { @@ -198,16 +195,12 @@ fn build(name: &str, ranks: &str) -> Result { }) } -fn parse_rank(line: &str) -> Result<(Vec, Rank), LoadError> { +fn parse_rank(line: &str) -> Result<(Vec, Rank), RankError> { let (token, rank) = line .split_once(' ') - .ok_or_else(|| LoadError::Ranks("missing rank".into()))?; - let bytes = STANDARD - .decode(token) - .map_err(|error| LoadError::Ranks(error.to_string()))?; - let rank = rank - .parse() - .map_err(|error: std::num::ParseIntError| LoadError::Ranks(error.to_string()))?; + .ok_or_else(|| RankError::MissingRank(line.to_owned()))?; + let bytes = STANDARD.decode(token).map_err(RankError::InvalidToken)?; + let rank = rank.parse().map_err(RankError::InvalidRank)?; Ok((bytes, rank)) } diff --git a/litellm-rust/crates/token-counter/Cargo.toml b/litellm-rust/crates/token-counter/Cargo.toml index 67e5bd60537..01ce4b6bac8 100644 --- a/litellm-rust/crates/token-counter/Cargo.toml +++ b/litellm-rust/crates/token-counter/Cargo.toml @@ -14,6 +14,7 @@ tiktoken = ["dep:litellm-token-counter-tiktoken"] [dependencies] indexmap = { version = "2.14.0", features = ["serde"] } itoa = "1.0" +litellm-error.workspace = true litellm-token-counter-fast = { workspace = true, optional = true } litellm-token-counter-huggingface = { workspace = true, optional = true } litellm-token-counter-tiktoken = { workspace = true, optional = true } diff --git a/litellm-rust/crates/token-counter/src/counter.rs b/litellm-rust/crates/token-counter/src/counter.rs index 6370d9d74f8..f3c16f93b84 100644 --- a/litellm-rust/crates/token-counter/src/counter.rs +++ b/litellm-rust/crates/token-counter/src/counter.rs @@ -1,12 +1,12 @@ use serde::Serialize; -use crate::Error; use crate::python_json; use crate::tools::format_function_definitions; use crate::types::{ ContentItem, CountableContentBlock, CountableRequest, Message, MessageContent, TextValue, ToolChoice, ToolDefinition, }; +use crate::{Error, Unsupported}; const TOKENS_PER_MESSAGE: usize = 3; const TOKENS_PER_NAME: usize = 1; @@ -53,7 +53,7 @@ impl TokenCounter { self.count_optional_text_value(request.query.as_ref())? + self.count_optional_text_value(request.documents.as_ref())? } else { - return Err(Error::MissingInput); + return Err(Unsupported::MissingInput.into()); }; Ok(InputTokenCount { model: request.model.clone(), @@ -96,7 +96,7 @@ impl TokenCounter { TextValue::Number(number) => match (number.as_i64(), number.as_u64()) { (Some(number), _) => self.count_text(itoa::Buffer::new().format(number)), (_, Some(number)) => self.count_text(itoa::Buffer::new().format(number)), - _ => Err(Error::FloatText), + _ => Err(Unsupported::FloatText.into()), }, TextValue::Text(text) => self.count_text(text), TextValue::List(items) => items @@ -143,7 +143,9 @@ impl TokenCounter { None => Ok(0), } } - ContentItem::Block(CountableContentBlock::Unsupported) => Err(Error::ContentBlock), + ContentItem::Block(CountableContentBlock::Unsupported) => { + Err(Unsupported::ContentBlock.into()) + } } } diff --git a/litellm-rust/crates/token-counter/src/error.rs b/litellm-rust/crates/token-counter/src/error.rs index 7a74138c074..89a97423c48 100644 --- a/litellm-rust/crates/token-counter/src/error.rs +++ b/litellm-rust/crates/token-counter/src/error.rs @@ -1,39 +1,66 @@ use std::string::FromUtf8Error; +use litellm_error::Source; use thiserror::Error as ThisError; #[derive(Debug, ThisError)] pub enum Error { - #[error("unsupported tokenizer: {0}")] - UnsupportedTokenizer(String), - #[error("failed to load tokenizer: {0}")] - Load(#[source] Box), - #[error("failed to load tokenizer: tiktoken rank file: {0}")] - Ranks(#[source] Box), - #[error("failed to load tokenizer: Unicode character classes are unavailable")] - UnicodeClasses, - #[error("unsupported by the rust token counter: request body could not be parsed: {0}")] - RequestParse(#[source] serde_json::Error), - #[error("unsupported by the rust token counter: request has no countable input")] - MissingInput, - #[error( - "unsupported by the rust token counter: float text values are counted by the python path" - )] - FloatText, - #[error( - "unsupported by the rust token counter: content block type is counted by the python path" - )] - ContentBlock, - #[error("unsupported by the rust token counter: array parameter without items")] - ArrayItems, - #[error("unsupported by the rust token counter: text value could not be serialized: {0}")] - JsonSerialization(#[source] serde_json::Error), - #[error("unsupported by the rust token counter: serialized text value is not UTF-8: {0}")] - JsonUtf8(#[source] FromUtf8Error), - #[error("tokenization failed: {0}")] - Encode(#[source] Box), - #[error("token decoding failed: {0}")] - Decode(String), - #[error("token counting task failed: {0}")] - Task(String), + #[cfg(feature = "fast")] + #[error(transparent)] + Fast(#[from] litellm_token_counter_fast::Error), + #[cfg(feature = "huggingface")] + #[error(transparent)] + HuggingFace(#[from] litellm_token_counter_huggingface::Error), + #[cfg(feature = "tiktoken")] + #[error(transparent)] + Tiktoken(#[from] litellm_token_counter_tiktoken::Error), + #[error("unsupported by the rust token counter")] + Unsupported(#[from] Unsupported), + #[error("token counting task failed")] + Task(#[source] Source), +} + +impl Error { + pub fn is_unsupported(&self) -> bool { + match self { + Self::Unsupported(_) => true, + #[cfg(feature = "fast")] + Self::Fast(_) => false, + #[cfg(feature = "huggingface")] + Self::HuggingFace(_) => false, + #[cfg(feature = "tiktoken")] + Self::Tiktoken(_) => false, + Self::Task(_) => false, + } + } + + pub fn is_load(&self) -> bool { + match self { + #[cfg(feature = "fast")] + Self::Fast(error) => error.is_load(), + #[cfg(feature = "huggingface")] + Self::HuggingFace(error) => error.is_load(), + #[cfg(feature = "tiktoken")] + Self::Tiktoken(error) => error.is_load(), + Self::Unsupported(_) | Self::Task(_) => false, + } + } +} + +#[derive(Debug, ThisError)] +pub enum Unsupported { + #[error("request body could not be parsed")] + RequestParse(#[source] serde_json::Error), + #[error("request has no countable input")] + MissingInput, + #[error("float text values are counted by the python path")] + FloatText, + #[error("content block type is counted by the python path")] + ContentBlock, + #[error("array parameter without items")] + ArrayItems, + #[error("text value could not be serialized")] + JsonSerialization(#[source] serde_json::Error), + #[error("serialized text value is not UTF-8")] + JsonUtf8(#[source] FromUtf8Error), } diff --git a/litellm-rust/crates/token-counter/src/fast.rs b/litellm-rust/crates/token-counter/src/fast.rs index d1f270dcdc9..3ef0021e355 100644 --- a/litellm-rust/crates/token-counter/src/fast.rs +++ b/litellm-rust/crates/token-counter/src/fast.rs @@ -1,4 +1,3 @@ -use litellm_token_counter_fast::Error as BackendError; pub use litellm_token_counter_fast::FastTokenizer; use crate::{Error, TextCodec, TokenCounter, Tokenizer}; @@ -53,17 +52,6 @@ impl Tokenizer for FastTokenizer { } } -impl From for Error { - fn from(error: BackendError) -> Self { - match error { - BackendError::Load(source) => Self::Load(source), - BackendError::Ranks(source) => Self::Ranks(source.into()), - BackendError::UnicodeClasses => Self::UnicodeClasses, - BackendError::Encode(source) => Self::Encode(source), - } - } -} - #[cfg(all(test, feature = "huggingface", feature = "tiktoken"))] mod tests { use super::*; diff --git a/litellm-rust/crates/token-counter/src/huggingface.rs b/litellm-rust/crates/token-counter/src/huggingface.rs index 43613fac8ea..5730e9ce873 100644 --- a/litellm-rust/crates/token-counter/src/huggingface.rs +++ b/litellm-rust/crates/token-counter/src/huggingface.rs @@ -1,4 +1,3 @@ -use litellm_token_counter_huggingface::Error as BackendError; pub use litellm_token_counter_huggingface::{ AddedToken, EncodeInput, Encoding, HuggingFaceTokenizer, InputSequence, PaddingDirection, PaddingParams, PaddingStrategy, TruncationDirection, TruncationParams, encoding_from_json, @@ -34,13 +33,3 @@ impl TextCodec for HuggingFaceTokenizer { HuggingFaceTokenizer::name(self) } } - -impl From for Error { - fn from(error: BackendError) -> Self { - match error { - BackendError::Load(source) => Self::Load(source), - BackendError::Encode(source) => Self::Encode(source), - BackendError::Decode(source) => Self::Decode(source.to_string()), - } - } -} diff --git a/litellm-rust/crates/token-counter/src/lib.rs b/litellm-rust/crates/token-counter/src/lib.rs index 84bf35ca2ba..c58b8cf372f 100644 --- a/litellm-rust/crates/token-counter/src/lib.rs +++ b/litellm-rust/crates/token-counter/src/lib.rs @@ -19,6 +19,6 @@ pub mod huggingface; pub mod tiktoken; pub use counter::{InputTokenCount, TokenCounter}; -pub use error::Error; +pub use error::{Error, Unsupported}; pub use tokenizer::{TextCodec, Tokenizer}; pub use types::CountableRequest; diff --git a/litellm-rust/crates/token-counter/src/python_json.rs b/litellm-rust/crates/token-counter/src/python_json.rs index 00a55ad300c..b5c074a24c7 100644 --- a/litellm-rust/crates/token-counter/src/python_json.rs +++ b/litellm-rust/crates/token-counter/src/python_json.rs @@ -6,8 +6,8 @@ use std::io::{self, Write}; use serde::Serialize; use serde_json::ser::{Formatter, Serializer}; -use super::Error; use super::types::TextValue; +use super::{Error, Unsupported}; pub(super) fn dumps(value: &TextValue) -> Result { let mut output = Vec::with_capacity(serialized_len(value)?); @@ -16,9 +16,9 @@ pub(super) fn dumps(value: &TextValue) -> Result { &mut output, PythonFormatter, )) - .map_err(Error::JsonSerialization)?; + .map_err(Unsupported::JsonSerialization)?; debug_assert_eq!(output.len(), output.capacity()); - String::from_utf8(output).map_err(Error::JsonUtf8) + String::from_utf8(output).map_err(|error| Unsupported::JsonUtf8(error).into()) } fn serialized_len(value: &TextValue) -> Result { @@ -29,7 +29,7 @@ fn serialized_len(value: &TextValue) -> Result { TextValue::Number(number) => match (number.as_i64(), number.as_u64()) { (Some(number), _) => Ok(unsigned_len(number.unsigned_abs()) + usize::from(number < 0)), (_, Some(number)) => Ok(unsigned_len(number)), - _ => Err(Error::FloatText), + _ => Err(Unsupported::FloatText.into()), }, TextValue::Text(text) => Ok(quoted_len(text)), TextValue::List(items) => items @@ -147,9 +147,9 @@ mod tests { let value = serde_json::from_str(input).expect("fixture parses"); let error = dumps(&value).expect_err("floats are declined"); - assert_eq!( - error.to_string(), - "unsupported by the rust token counter: float text values are counted by the python path" - ); + assert!(matches!( + error, + crate::Error::Unsupported(crate::Unsupported::FloatText) + )); } } diff --git a/litellm-rust/crates/token-counter/src/tiktoken.rs b/litellm-rust/crates/token-counter/src/tiktoken.rs index b95621f3ccc..136280b6115 100644 --- a/litellm-rust/crates/token-counter/src/tiktoken.rs +++ b/litellm-rust/crates/token-counter/src/tiktoken.rs @@ -1,4 +1,3 @@ -use litellm_token_counter_tiktoken::{LoadError, UnsupportedTokenizer}; pub use litellm_token_counter_tiktoken::{TiktokenTokenizer, Vocabulary, encoding_for_model}; use crate::{Error, TextCodec, TokenCounter, Tokenizer}; @@ -23,25 +22,10 @@ impl TextCodec for TiktokenTokenizer { } fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result { - TiktokenTokenizer::decode(self, ids).map_err(|error| Error::Decode(error.to_string())) + TiktokenTokenizer::decode(self, ids).map_err(Error::from) } fn name(&self) -> &str { TiktokenTokenizer::name(self) } } - -impl From for Error { - fn from(error: UnsupportedTokenizer) -> Self { - Self::UnsupportedTokenizer(error.0) - } -} - -impl From for Error { - fn from(error: LoadError) -> Self { - match error { - LoadError::Unsupported(error) => error.into(), - LoadError::Ranks(source) => Self::Ranks(source.into()), - } - } -} diff --git a/litellm-rust/crates/token-counter/src/tools.rs b/litellm-rust/crates/token-counter/src/tools.rs index 22150a4e4b1..a5a3fdbfabf 100644 --- a/litellm-rust/crates/token-counter/src/tools.rs +++ b/litellm-rust/crates/token-counter/src/tools.rs @@ -3,8 +3,8 @@ use std::fmt::Write; -use super::Error; use super::types::{EnumValue, Schema, SchemaType, ToolDefinition}; +use super::{Error, Unsupported}; pub(super) fn format_function_definitions(tools: &[ToolDefinition]) -> Result { ToolFormatter::new(tools.len()).format(tools) @@ -117,7 +117,7 @@ impl ToolFormatter { None => self.output.push_str("number"), }, "array" => { - let items = props.items.as_deref().ok_or(Error::ArrayItems)?; + let items = props.items.as_deref().ok_or(Unsupported::ArrayItems)?; self.write_type(items, indent)?; self.output.push_str("[]"); } @@ -298,7 +298,7 @@ mod tests { assert!(matches!( format_function_definitions(&tools), - Err(Error::ArrayItems) + Err(Error::Unsupported(Unsupported::ArrayItems)) )); } } diff --git a/litellm-rust/crates/token-counter/src/types.rs b/litellm-rust/crates/token-counter/src/types.rs index c1236f94d9a..7e1513d7706 100644 --- a/litellm-rust/crates/token-counter/src/types.rs +++ b/litellm-rust/crates/token-counter/src/types.rs @@ -5,7 +5,7 @@ use serde::de::{MapAccess, SeqAccess, Visitor}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Number; -use super::Error; +use super::{Error, Unsupported}; /// The parts of a request body the host's budget counter reads. Chat and /// Anthropic Messages bodies carry `messages`; completions carry `prompt`; @@ -33,7 +33,7 @@ pub struct CountableRequest { impl CountableRequest { pub fn parse(body: &[u8]) -> Result { - serde_json::from_slice(body).map_err(Error::RequestParse) + serde_json::from_slice(body).map_err(|error| Unsupported::RequestParse(error).into()) } } diff --git a/litellm-rust/crates/token-counter/tests/token_counter.rs b/litellm-rust/crates/token-counter/tests/token_counter.rs index 542bd4a1fc4..cc18bad5402 100644 --- a/litellm-rust/crates/token-counter/tests/token_counter.rs +++ b/litellm-rust/crates/token-counter/tests/token_counter.rs @@ -2,7 +2,7 @@ use rstest::rstest; #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))] use litellm_token_counter::TokenCounter; -use litellm_token_counter::{CountableRequest, Error}; +use litellm_token_counter::{CountableRequest, Error, Unsupported}; #[cfg(any(feature = "fast", feature = "huggingface"))] mod json { @@ -100,7 +100,12 @@ mod json { let request = CountableRequest::parse(body).expect("shape parses"); assert!(matches!( counter(load).count_request(&request), - Err(Error::MissingInput | Error::FloatText | Error::ContentBlock | Error::ArrayItems) + Err(Error::Unsupported( + Unsupported::MissingInput + | Unsupported::FloatText + | Unsupported::ContentBlock + | Unsupported::ArrayItems + )) )); } @@ -139,7 +144,7 @@ mod json { } fn assert_loading_a_bad_tokenizer_is_a_load_error(load: JsonLoader) { - assert!(matches!(load("{}"), Err(Error::Load(_)))); + assert!(load("{}").is_err_and(|error| error.is_load())); } macro_rules! json_backend_tests { @@ -206,7 +211,7 @@ mod json { let counter = $loader(&tokenizer.to_string(false).unwrap()).unwrap(); let request = CountableRequest::parse(br#"{"prompt":"hello"}"#).unwrap(); let error = counter.count_request(&request).unwrap_err(); - assert!(matches!(error, Error::Encode(_))); + assert!(!error.is_load() && !error.is_unsupported()); assert_eq!(error.source().unwrap().to_string(), expected.to_string()); } @@ -250,7 +255,7 @@ mod json { fn shapes_outside_the_mirror_are_declined_at_parse(#[case] body: &[u8]) { assert!(matches!( CountableRequest::parse(body), - Err(Error::RequestParse(_)) + Err(Error::Unsupported(Unsupported::RequestParse(_))) )); } @@ -443,7 +448,7 @@ mod tiktoken { #[case] rank_file: &str, #[values(CL100K, O200K)] encoding: TiktokenEncoding, ) { - assert!(matches!((encoding.load)(rank_file), Err(Error::Ranks(_)))); + assert!((encoding.load)(rank_file).is_err_and(|error| error.is_load())); } } @@ -452,6 +457,7 @@ mod tiktoken { fn unsupported_encoding_reaches_the_counter_caller() { assert!(matches!( TokenCounter::from_tiktoken("unknown-encoding"), - Err(Error::UnsupportedTokenizer(name)) if name == "unknown-encoding" + Err(error) if error.is_load() + && litellm_error::Report::new(&error).to_string().contains("unknown-encoding") )); }