chore(auto-router): merge main and preserve JEV configuration

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.circleci/config.yml

@@ -1508,7 +1508,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
+            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_3_13:
     docker:
@@ -1532,7 +1532,7 @@ jobs:
       - run:
           name: Run tests
           command: |
-            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
+            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not legacy_resolver"
 
   installing_litellm_on_python_v2_migration_resolver:
     docker:
@@ -1561,10 +1561,11 @@ jobs:
           url: tcp://localhost:5432
           timeout: "60"
       - run:
-          name: Run v2 migration resolver proxy smoke test
+          name: Run both migration resolvers against Postgres
           command: |
             uv run --no-sync python -m pytest -vv \
-              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver
+              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings \
+              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_legacy_resolver
 
   helm_chart_testing:
     machine:

.github/workflows/test-e2e-changed.yml

@@ -175,6 +175,8 @@ jobs:
         env:
           TESTS: ${{ needs.detect.outputs.tests }}
           E2E_FIXTURE_MODE: live
+          E2E_PROVIDER_EDGE_HOST_REACHABLE: '1'
+          COLUMNS: '400'
         run: |
           umask 077
           read -r -a test_files <<< "${TESTS}"
@@ -189,6 +191,7 @@ jobs:
             uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"
             verified=$?
             set -e
+            grep -E '^(FAILED|ERROR) ' "${log}" || true
             grep -E '^=+ .* in [0-9.]+s( \([0-9:]+\))? =+$' "${log}" | tail -n 1
             echo "::endgroup::"
             if [ "${status}" = "5" ]; then

.github/workflows/test-rust.yml

@@ -120,6 +120,20 @@ jobs:
 
       - run: cargo test --workspace --doc --locked
 
+      - name: Test token counter feature combinations
+        run: |
+          for features in '' fast huggingface tiktoken fast,huggingface fast,tiktoken huggingface,tiktoken fast,huggingface,tiktoken; do
+            cargo test -p litellm-token-counter --locked --no-default-features --features "$features"
+            cargo check -p litellm-python-bridge --locked --no-default-features --features "abi3${features:+,$features}"
+          done
+
+      - name: Test secret manager feature combinations
+        run: |
+          cargo test -p litellm-auth-gcp --locked --no-default-features
+          for features in '' aws google aws,google; do
+            cargo test -p litellm-secrets --locked --no-default-features --features "$features"
+          done
+
   rust-wheel:
     runs-on: ubuntu-latest
     timeout-minutes: 30

.github/workflows/test-unit.yml

@@ -51,7 +51,7 @@ jobs:
         include:
           - shard: mcp-integration
             artifact-name: mcp-integration
-            test-path: "tests/mcp_tests"
+            test-path: "tests/mcp_tests tests/test_litellm/experimental_mcp_client"
             workers: 2
             reruns: 0
             timeout-minutes: 20
@@ -113,7 +113,6 @@ jobs:
               tests/test_litellm/compression
               tests/test_litellm/containers
               tests/test_litellm/endpoints
-              tests/test_litellm/experimental_mcp_client
               tests/test_litellm/models
               tests/test_litellm/repositories
               tests/test_litellm/images

helm/litellm/templates/backend/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: backend
 spec:
+  {{- if and (not .Values.backend.hpa.enabled) (not (kindIs "invalid" .Values.backend.replicaCount)) }}
+  replicas: {{ .Values.backend.replicaCount }}
+  {{- end }}
   {{- with .Values.backend.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/templates/gateway/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: gateway
 spec:
+  {{- if and (not .Values.gateway.hpa.enabled) (not (kindIs "invalid" .Values.gateway.replicaCount)) }}
+  replicas: {{ .Values.gateway.replicaCount }}
+  {{- end }}
   {{- with .Values.gateway.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/templates/ui/deployment.yaml

@@ -7,6 +7,9 @@ metadata:
     {{- include "litellm.commonLabels" . | nindent 4 }}
     app.kubernetes.io/component: ui
 spec:
+  {{- if and (not .Values.ui.hpa.enabled) (not (kindIs "invalid" .Values.ui.replicaCount)) }}
+  replicas: {{ .Values.ui.replicaCount }}
+  {{- end }}
   {{- with .Values.ui.strategy }}
   strategy:
     {{- toYaml . | nindent 4 }}

helm/litellm/tests/replica_count_tests.yaml

@@ -0,0 +1,100 @@
+suite: test fixed replica count when HPA is disabled
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: gateway renders replicaCount into spec.replicas when its HPA is disabled
+    template: gateway/deployment.yaml
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 3
+    asserts:
+      - isKind:
+          of: Deployment
+      - equal:
+          path: spec.replicas
+          value: 3
+
+  - it: backend renders replicaCount into spec.replicas when its HPA is disabled
+    template: backend/deployment.yaml
+    set:
+      backend.hpa.enabled: false
+      backend.replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+
+  - it: ui renders replicaCount into spec.replicas when its HPA is disabled
+    template: ui/deployment.yaml
+    set:
+      ui.hpa.enabled: false
+      ui.replicaCount: 2
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 2
+
+  - it: replicaCount 0 scales the gateway to zero instead of being treated as unset
+    template: gateway/deployment.yaml
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 0
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 0
+
+  - it: a component with HPA disabled but no replicaCount set keeps omitting spec.replicas, so upgrades do not reset a hand-scaled Deployment
+    set:
+      gateway.hpa.enabled: false
+      backend.hpa.enabled: false
+      ui.hpa.enabled: false
+    asserts:
+      - notExists:
+          path: spec.replicas
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: ui/deployment.yaml
+
+  - it: every component omits spec.replicas when its HPA is enabled, so the autoscaler owns the count
+    set:
+      gateway.hpa.enabled: true
+      gateway.replicaCount: 3
+      backend.hpa.enabled: true
+      backend.replicaCount: 3
+      ui.hpa.enabled: true
+      ui.replicaCount: 3
+    asserts:
+      - notExists:
+          path: spec.replicas
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: ui/deployment.yaml
+
+  - it: a component with HPA disabled renders replicas while a sibling with HPA enabled does not
+    set:
+      gateway.hpa.enabled: false
+      gateway.replicaCount: 4
+      backend.hpa.enabled: true
+      backend.replicaCount: 4
+    asserts:
+      - equal:
+          path: spec.replicas
+          value: 4
+        template: gateway/deployment.yaml
+      - notExists:
+          path: spec.replicas
+        template: backend/deployment.yaml

helm/litellm/values.yaml

@@ -397,6 +397,11 @@ gateway:
   #   failureThreshold: 30
   #   periodSeconds: 10
   startupProbe: {}
+  # Optional fixed pod count, rendered into the Deployment's spec.replicas only
+  # when hpa.enabled is false. Unset by default so an existing Deployment keeps
+  # its current count; with the HPA on, the autoscaler owns the count, e.g.:
+  #   replicaCount: 3
+  replicaCount:
   hpa:
     enabled: true
     minReplicas: 1
@@ -524,6 +529,8 @@ backend:
   strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
+  # Same semantics as gateway.replicaCount.
+  replicaCount:
   hpa:
     enabled: true
     minReplicas: 1
@@ -590,6 +597,8 @@ ui:
   strategy: {}
   # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
   startupProbe: {}
+  # Same semantics as gateway.replicaCount.
+  replicaCount:
   hpa:
     enabled: false
     minReplicas: 1

litellm-rust/Cargo.toml

@@ -14,9 +14,14 @@ litellm-host = { path = "crates/host" }
 litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
 litellm-framing = { path = "crates/framer" }
 litellm-auth = { path = "crates/auth" }
+litellm-auth-types = { path = "crates/auth-types" }
 litellm-auth-aws = { path = "crates/auth-aws" }
 litellm-auth-azure = { path = "crates/auth-azure" }
 litellm-auth-gcp = { path = "crates/auth-gcp" }
+litellm-secrets = { path = "crates/secrets" }
+litellm-secrets-types = { path = "crates/secrets-types" }
+litellm-secrets-aws = { path = "crates/secrets-aws" }
+litellm-secrets-google = { path = "crates/secrets-google" }
 litellm-http = { path = "crates/http" }
 litellm-llms = { path = "crates/llms" }
 litellm-types = { path = "crates/types" }
@@ -24,10 +29,15 @@ litellm-core-utils = { path = "crates/core-utils" }
 litellm-cache = { path = "crates/cache" }
 litellm-cache-memory = { path = "crates/cache-memory" }
 litellm-token-counter = { path = "crates/token-counter" }
+litellm-token-counter-fast = { path = "crates/token-counter-fast" }
+litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" }
+litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
 litellm-host-python = { path = "crates/host-python" }
 
 bytes = "1"
 http = "1"
+google-cloud-auth = { version = "1.16.0", default-features = false }
+jsonwebtoken = { version = "11.1.0", default-features = false }
 hyper-util = { version = "0.1.20", default-features = false, features = ["client-proxy"] }
 proptest = "1.7.0"
 pyo3 = "0.29.2"
@@ -45,6 +55,8 @@ serde_with = { version = "=3.16.1", default-features = false, features = ["std",
 sha2 = "0.10"
 subtle = "2"
 thiserror = "2.0"
+tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
+tiktoken-rs = "0.12.0"
 tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "net"] }
 tokio-tungstenite = { version = "0.24", default-features = false, features = ["connect", "rustls-tls-native-roots"] }
 futures-util = { version = "0.3", default-features = false, features = ["sink", "std"] }

litellm-rust/crates/auth-aws/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-litellm-auth.workspace = true
+litellm-auth-types.workspace = true
 litellm-http.workspace = true
 
 moka = { workspace = true, features = ["sync"] }

litellm-rust/crates/auth-aws/src/constants.rs

@@ -3,6 +3,8 @@ pub const AWS_SECRET_ACCESS_KEY: &str = "AWS_SECRET_ACCESS_KEY";
 pub const AWS_SESSION_TOKEN: &str = "AWS_SESSION_TOKEN";
 pub const AWS_REGION_NAME: &str = "AWS_REGION_NAME";
 pub const AWS_REGION: &str = "AWS_REGION";
+pub const AWS_DEFAULT_REGION: &str = "AWS_DEFAULT_REGION";
+pub const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "AWS_BEDROCK_RUNTIME_ENDPOINT";
 pub const AWS_SESSION_NAME: &str = "AWS_SESSION_NAME";
 pub const AWS_PROFILE_NAME: &str = "AWS_PROFILE_NAME";
 pub const AWS_ROLE_NAME: &str = "AWS_ROLE_NAME";

litellm-rust/crates/auth-aws/src/error.rs

@@ -22,7 +22,7 @@ pub enum Error {
     AwsMissingWebIdentityCredentials,
 }
 
-impl From<Error> for litellm_auth::Error {
+impl From<Error> for litellm_auth_types::Error {
     fn from(error: Error) -> Self {
         Self::ProviderAuthentication(error.to_string())
     }
@@ -34,11 +34,11 @@ mod tests {
 
     #[test]
     fn converts_to_shared_auth_error_without_losing_context() {
-        let error = litellm_auth::Error::from(Error::AwsProfile("profile not found".into()));
+        let error = litellm_auth_types::Error::from(Error::AwsProfile("profile not found".into()));
 
         assert_eq!(
             error,
-            litellm_auth::Error::ProviderAuthentication(
+            litellm_auth_types::Error::ProviderAuthentication(
                 "AWS profile credentials failed: profile not found".into()
             )
         );

litellm-rust/crates/auth-azure/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
-litellm-auth.workspace = true
+litellm-auth-types.workspace = true
 
 moka.workspace = true
 serde_json.workspace = true

litellm-rust/crates/auth-azure/src/credential_provider_cache.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 use azure_core::credentials::TokenCredential;
 use moka::future::Cache;
 
-use litellm_auth::Error;
+use litellm_auth_types::Error;
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub(crate) struct AzureCredentialProviderCacheKey {

litellm-rust/crates/auth-azure/src/native.rs

@@ -12,8 +12,8 @@ use azure_identity::{
 };
 use sha2::{Digest, Sha256};
 
-use litellm_auth::Error;
-use litellm_auth::{InputSource, ResolvedCredential, SecretValue, Sourced};
+use litellm_auth_types::Error;
+use litellm_auth_types::{InputSource, ResolvedCredential, SecretValue, Sourced};
 
 use super::credential_provider_cache::{
     AzureCredentialProviderCache, AzureCredentialProviderCacheKey,
@@ -484,7 +484,7 @@ mod tests {
     use azure_core::{Bytes, Result};
 
     use super::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest};
-    use litellm_auth::{InputSource, SecretValue, Sourced};
+    use litellm_auth_types::{InputSource, SecretValue, Sourced};
 
     fn deployment<T>(value: T) -> Sourced<T> {
         Sourced::new(value, InputSource::Deployment)
@@ -649,7 +649,7 @@ mod tests {
 
         assert!(matches!(
             error,
-            litellm_auth::Error::MixedAzureCredentialSources
+            litellm_auth_types::Error::MixedAzureCredentialSources
         ));
     }
 
@@ -679,7 +679,10 @@ mod tests {
                 authority,
             ))
             .unwrap_err();
-            assert!(matches!(error, litellm_auth::Error::InvalidAzureAuthority));
+            assert!(matches!(
+                error,
+                litellm_auth_types::Error::InvalidAzureAuthority
+            ));
         }
     }
 }

litellm-rust/crates/auth-azure/src/resolve.rs

@@ -1,5 +1,5 @@
-use litellm_auth::Error;
-use litellm_auth::{
+use litellm_auth_types::Error;
+use litellm_auth_types::{
     CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential,
     SecretValue, Sourced, TokenProviderHandle,
 };
@@ -451,9 +451,9 @@ mod tests {
     };
     use crate::native::ValidatedAzureRequest;
     use crate::types::AzureAuthInputs;
-    use litellm_auth::Error;
-    use litellm_auth::ResolvedCredential;
-    use litellm_auth::{
+    use litellm_auth_types::Error;
+    use litellm_auth_types::ResolvedCredential;
+    use litellm_auth_types::{
         CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialRef,
         CredentialResolver, CredentialResolverHandle, InputSource, SecretValue, Sourced,
     };
@@ -661,8 +661,8 @@ mod tests {
     #[derive(Debug)]
     struct CallerToken(&'static str);
 
-    impl litellm_auth::TokenProvider for CallerToken {
-        fn acquire(&self) -> litellm_auth::TokenFuture<'_> {
+    impl litellm_auth_types::TokenProvider for CallerToken {
+        fn acquire(&self) -> litellm_auth_types::TokenFuture<'_> {
             Box::pin(async move {
                 Ok(ResolvedCredential::AccessToken {
                     token: SecretValue::new(self.0),
@@ -675,7 +675,7 @@ mod tests {
     fn caller_inputs(token: &'static str) -> AzureAuthInputs {
         let params = json!({"azure_ad_token": "static-token"});
         AzureAuthInputs {
-            azure_ad_token_provider: Some(litellm_auth::TokenProviderHandle::new(Arc::new(
+            azure_ad_token_provider: Some(litellm_auth_types::TokenProviderHandle::new(Arc::new(
                 CallerToken(token),
             ))),
             ..AzureAuthInputs::from_optional_params(params.as_object().unwrap()).unwrap()

litellm-rust/crates/auth-azure/src/types.rs

@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
 
-use litellm_auth::{
+use litellm_auth_types::{
     CredentialResolverHandle, Error, InputSource, SecretValue, Sourced, TokenProviderHandle,
 };
 use serde_json::{Map, Value};
@@ -126,7 +126,7 @@ fn source_for(sources: &BTreeMap<String, InputSource>, name: &str) -> InputSourc
 mod tests {
     use std::collections::BTreeMap;
 
-    use litellm_auth::{InputSource, Sourced};
+    use litellm_auth_types::{InputSource, Sourced};
     use serde_json::json;
 
     use super::{AzureAuthInputs, AzureCredentialType, ConfigValue};

litellm-rust/crates/auth-gcp/Cargo.toml

@@ -5,8 +5,11 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[features]
+google-sdk = ["dep:google-cloud-auth", "dep:http"]
+
 [dependencies]
-litellm-auth.workspace = true
+litellm-auth-types.workspace = true
 
 moka.workspace = true
 serde_json.workspace = true
@@ -14,3 +17,5 @@ sha2.workspace = true
 tokio.workspace = true
 
 gcp_auth = "0.12.7"
+google-cloud-auth = { workspace = true, optional = true }
+http = { workspace = true, optional = true }

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -1,13 +1,18 @@
 use std::{collections::BTreeMap, future::Future, path::Path, pin::Pin, sync::Arc};
 
 use gcp_auth::{CustomServiceAccount, TokenProvider};
-use litellm_auth::{
+use litellm_auth_types::{
     CredentialPlacement, Error, InputSource, SecretValue, Sourced, http::apply_credential,
 };
 use moka::future::Cache;
 use serde_json::{Map, Value};
 use sha2::{Digest, Sha256};
 
+#[cfg(feature = "google-sdk")]
+mod sdk;
+#[cfg(feature = "google-sdk")]
+pub use sdk::GoogleCredentials;
+
 const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
 const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
 const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
@@ -26,19 +31,31 @@ pub struct VertexConfig {
 }
 
 impl VertexConfig {
+    pub fn new(
+        credentials: Option<Sourced<SecretValue>>,
+        project_id: Option<String>,
+        location: Option<String>,
+    ) -> Self {
+        Self {
+            credentials: credentials.filter(|value| !value.value().expose().trim().is_empty()),
+            project_id: project_id.filter(|value| !value.trim().is_empty()),
+            location: location.filter(|value| !value.trim().is_empty()),
+        }
+    }
+
     pub fn from_sourced_optional_params(
         params: &Map<String, Value>,
         sources: &BTreeMap<String, InputSource>,
     ) -> Result<Self, Error> {
-        Ok(Self {
-            credentials: optional_credentials(
+        Ok(Self::new(
+            optional_credentials(
                 params,
                 sources,
                 &["vertex_credentials", "vertex_ai_credentials"],
             )?,
-            project_id: optional_string(params, &["vertex_project", "vertex_ai_project"])?,
-            location: optional_string(params, &["vertex_location", "vertex_ai_location"])?,
-        })
+            optional_string(params, &["vertex_project", "vertex_ai_project"])?,
+            optional_string(params, &["vertex_location", "vertex_ai_location"])?,
+        ))
     }
 
     pub fn or_configured(self, project_id: Option<&str>, location: Option<&str>) -> Self {
@@ -469,6 +486,39 @@ mod tests {
         assert_eq!(config.location(), Some("alias-location"));
     }
 
+    #[test]
+    fn typed_config_preserves_source_and_empty_value_fallback() {
+        let configured = VertexConfig::new(
+            Some(Sourced::new(
+                SecretValue::new("inline-json"),
+                InputSource::Request,
+            )),
+            Some("project".into()),
+            Some("location".into()),
+        );
+        assert!(matches!(
+            credential_source(&configured, &|_| Some("environment-json".into())),
+            CredentialSource::Inline(value) if value.expose() == "inline-json"
+        ));
+        let empty = VertexConfig::new(
+            Some(Sourced::new(SecretValue::new(" "), InputSource::Request)),
+            Some(" ".into()),
+            Some(" ".into()),
+        );
+        assert!(matches!(
+            credential_source(&empty, &|_| None),
+            CredentialSource::Adc
+        ));
+        assert_eq!(
+            get_vertex_ai_project(&empty, &|_| Some("env-project".into())).as_deref(),
+            Some("env-project")
+        );
+        assert_eq!(
+            get_vertex_ai_location(&empty, &|_| Some("env-location".into())).as_deref(),
+            Some("env-location")
+        );
+    }
+
     #[test]
     fn project_and_location_prefer_input_then_environment() {
         let configured =

litellm-rust/crates/auth-gcp/src/sdk.rs

@@ -0,0 +1,106 @@
+use std::sync::Arc;
+
+use google_cloud_auth::credentials::{CacheableResource, CredentialsProvider, EntityTag};
+use google_cloud_auth::errors::CredentialsError;
+use http::{Extensions, HeaderMap, HeaderName, HeaderValue};
+use litellm_auth_types::Error;
+
+use crate::{VertexAuth, VertexConfig};
+
+type EnvironmentLookup = dyn Fn(&str) -> Option<String> + Send + Sync;
+
+pub struct GoogleCredentials {
+    auth: VertexAuth,
+    config: VertexConfig,
+    environment: Arc<EnvironmentLookup>,
+}
+
+impl GoogleCredentials {
+    pub fn new(config: VertexConfig, environment: Arc<EnvironmentLookup>) -> Self {
+        Self {
+            auth: VertexAuth::default(),
+            config,
+            environment,
+        }
+    }
+
+    pub async fn request_headers(&self) -> Result<HeaderMap, Error> {
+        let response = self
+            .auth
+            .validate_environment(Vec::new(), None, &self.config, &|name| {
+                (self.environment)(name)
+            })
+            .await?;
+        response
+            .headers
+            .into_iter()
+            .map(|(key, value)| {
+                let name =
+                    HeaderName::from_bytes(key.as_bytes()).map_err(|_| Error::InvalidHeader)?;
+                let value = HeaderValue::from_str(&value).map_err(|_| Error::InvalidHeader)?;
+                Ok((name, value))
+            })
+            .collect()
+    }
+}
+
+impl CredentialsProvider for GoogleCredentials {
+    async fn headers(
+        &self,
+        _: Extensions,
+    ) -> Result<CacheableResource<HeaderMap>, CredentialsError> {
+        self.request_headers()
+            .await
+            .map(|data| CacheableResource::New {
+                entity_tag: EntityTag::new(),
+                data,
+            })
+            .map_err(|_| CredentialsError::from_msg(false, "Google authentication failed"))
+    }
+
+    async fn universe_domain(&self) -> Option<String> {
+        None
+    }
+}
+
+impl std::fmt::Debug for GoogleCredentials {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("GoogleCredentials").finish_non_exhaustive()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[tokio::test]
+    async fn sdk_and_http_credentials_share_token_resolution_and_redaction() {
+        let credentials = GoogleCredentials::new(
+            VertexConfig::new(None, Some("project".into()), None),
+            Arc::new(|name| (name == "VERTEX_AI_API_KEY").then(|| "private-token".into())),
+        );
+        let direct = credentials.request_headers().await.unwrap();
+        let CacheableResource::New { data, .. } =
+            credentials.headers(Extensions::new()).await.unwrap()
+        else {
+            panic!("first request did not return headers");
+        };
+        assert_eq!(direct, data);
+        assert_eq!(data[http::header::AUTHORIZATION], "Bearer private-token");
+        assert!(!format!("{credentials:?}").contains("private-token"));
+    }
+
+    #[tokio::test]
+    async fn invalid_token_headers_return_a_redacted_sdk_error() {
+        let credentials = GoogleCredentials::new(
+            VertexConfig::new(None, Some("project".into()), None),
+            Arc::new(|name| (name == "VERTEX_AI_API_KEY").then(|| "private\nvalue".into())),
+        );
+        assert_eq!(
+            credentials.request_headers().await.unwrap_err(),
+            Error::InvalidHeader
+        );
+        let error = credentials.headers(Extensions::new()).await.unwrap_err();
+        assert!(!format!("{error:?}").contains("private"));
+    }
+}

litellm-rust/crates/auth-types/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "litellm-auth-types"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+serde.workspace = true
+subtle.workspace = true
+thiserror.workspace = true
+veil.workspace = true
+
+[dev-dependencies]
+tokio.workspace = true

litellm-rust/crates/auth-types/src/credential.rs

@@ -5,9 +5,7 @@ use std::sync::Arc;
 
 use veil::Redact;
 
-use crate::Error;
-
-use super::{ResolvedCredential, SecretValue, TokenProviderHandle};
+use crate::{Error, ResolvedCredential, SecretValue, TokenProviderHandle};
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum CredentialFileRef {

litellm-rust/crates/auth-types/src/http.rs

@@ -40,9 +40,6 @@ pub fn apply_credential(
     )
 }
 
-/// How the upstream call is authenticated. API-key strategies become headers
-/// in `prepare`; SigV4 covers the serialized body, so it is applied where the
-/// outbound request is built.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum RequestAuth {
     Header {

litellm-rust/crates/auth-types/src/lib.rs

@@ -0,0 +1,57 @@
+#![forbid(unsafe_code)]
+
+mod credential;
+mod error;
+pub mod http;
+mod policy;
+mod secret;
+mod token;
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum InputSource {
+    Request,
+    #[default]
+    Deployment,
+    Environment,
+}
+
+#[derive(Clone, Debug, Default, Eq, PartialEq)]
+pub struct Sourced<T> {
+    value: T,
+    source: InputSource,
+}
+
+impl<T> Sourced<T> {
+    pub fn new(value: T, source: InputSource) -> Self {
+        Self { value, source }
+    }
+
+    pub fn value(&self) -> &T {
+        &self.value
+    }
+
+    pub fn source(&self) -> InputSource {
+        self.source
+    }
+
+    pub fn into_value(self) -> T {
+        self.value
+    }
+
+    pub fn map<U>(self, map: impl FnOnce(T) -> U) -> Sourced<U> {
+        Sourced::new(map(self.value), self.source)
+    }
+}
+
+pub use credential::{
+    CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan,
+    CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle,
+};
+pub use error::Error;
+pub use http::{CredentialPlacement, RequestAuth};
+pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
+pub use secret::SecretValue;
+pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};

litellm-rust/crates/auth-types/src/policy.rs

@@ -1,7 +1,5 @@
-use crate::Error;
-
-use super::http::apply_credential;
-use super::{CredentialPlacement, ResolvedCredential};
+use crate::http::apply_credential;
+use crate::{CredentialPlacement, Error, ResolvedCredential};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum CredentialPlanKind {

litellm-rust/crates/auth-types/src/token.rs

@@ -5,9 +5,7 @@ use std::time::SystemTime;
 
 use veil::Redact;
 
-use crate::Error;
-
-use super::secret::SecretValue;
+use crate::{Error, SecretValue};
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum ResolvedCredential {

litellm-rust/crates/auth/Cargo.toml

@@ -5,11 +5,14 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
-[dependencies]
-serde.workspace = true
-subtle.workspace = true
-thiserror.workspace = true
-veil.workspace = true
+[features]
+default = []
+aws = ["dep:litellm-auth-aws"]
+azure = ["dep:litellm-auth-azure"]
+gcp = ["dep:litellm-auth-gcp"]
 
-[dev-dependencies]
-tokio.workspace = true
+[dependencies]
+litellm-auth-types.workspace = true
+litellm-auth-aws = { workspace = true, optional = true }
+litellm-auth-azure = { workspace = true, optional = true }
+litellm-auth-gcp = { workspace = true, optional = true }

litellm-rust/crates/auth/src/lib.rs

@@ -1,55 +1,10 @@
-mod credential;
-mod error;
-pub mod http;
-mod policy;
-mod secret;
-mod token;
+#![forbid(unsafe_code)]
 
-use serde::{Deserialize, Serialize};
+pub use litellm_auth_types::*;
 
-#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, Hash, PartialEq, Serialize)]
-#[serde(rename_all = "snake_case")]
-pub enum InputSource {
-    Request,
-    #[default]
-    Deployment,
-    Environment,
-}
-
-#[derive(Clone, Debug, Default, Eq, PartialEq)]
-pub struct Sourced<T> {
-    value: T,
-    source: InputSource,
-}
-
-impl<T> Sourced<T> {
-    pub fn new(value: T, source: InputSource) -> Self {
-        Self { value, source }
-    }
-
-    pub fn value(&self) -> &T {
-        &self.value
-    }
-
-    pub fn source(&self) -> InputSource {
-        self.source
-    }
-
-    pub fn into_value(self) -> T {
-        self.value
-    }
-
-    pub fn map<U>(self, map: impl FnOnce(T) -> U) -> Sourced<U> {
-        Sourced::new(map(self.value), self.source)
-    }
-}
-
-pub use credential::{
-    CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan,
-    CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle,
-};
-pub use error::Error;
-pub use http::{CredentialPlacement, RequestAuth};
-pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
-pub use secret::SecretValue;
-pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
+#[cfg(feature = "aws")]
+pub use litellm_auth_aws as aws;
+#[cfg(feature = "azure")]
+pub use litellm_auth_azure as azure;
+#[cfg(feature = "gcp")]
+pub use litellm_auth_gcp as gcp;

litellm-rust/crates/auth/tests/facade.rs

@@ -0,0 +1,33 @@
+use litellm_auth::{
+    CredentialPlacement, CredentialPlanKind, CredentialRule, ExistingHeaderBehavior,
+    ProviderAuthPolicy, ResolvedCredential, SecretValue,
+};
+
+const RULES: &[CredentialRule] = &[CredentialRule {
+    kind: CredentialPlanKind::Static,
+    placement: CredentialPlacement::Header("x-api-key"),
+}];
+
+#[test]
+fn facade_applies_shared_auth_policy() {
+    let policy = ProviderAuthPolicy {
+        rules: RULES,
+        accepted_existing_headers: &["x-api-key"],
+        existing_header_behavior: ExistingHeaderBehavior::Preserve,
+        scope: None,
+        audience: None,
+    };
+
+    let headers = policy
+        .apply(
+            Vec::new(),
+            CredentialPlanKind::Static,
+            &ResolvedCredential::Static(SecretValue::new("secret")),
+        )
+        .expect("facade policy applies");
+
+    assert_eq!(
+        headers,
+        vec![("x-api-key".to_string(), "secret".to_string())]
+    );
+}

litellm-rust/crates/python-bridge/Cargo.toml

@@ -10,10 +10,13 @@ name = "_native"
 crate-type = ["cdylib"]
 
 [features]
-default = ["abi3"]
+default = ["abi3", "fast"]
 abi3 = ["pyo3/abi3-py310"]
 extension-module = ["pyo3/extension-module"]
 panic-test = []
+fast = ["litellm-token-counter/fast"]
+huggingface = ["litellm-token-counter/huggingface"]
+tiktoken = ["litellm-token-counter/tiktoken"]
 
 [dependencies]
 bytes.workspace = true
@@ -26,7 +29,7 @@ litellm-http.workspace = true
 litellm-llms.workspace = true
 litellm-types.workspace = true
 litellm-host-python.workspace = true
-litellm-token-counter.workspace = true
+litellm-token-counter = { path = "../token-counter", default-features = false }
 pyo3.workspace = true
 pyo3-async-runtimes.workspace = true
 serde_json.workspace = true

litellm-rust/crates/python-bridge/src/token_counter.rs

@@ -1,6 +1,11 @@
-use std::{num::NonZero, sync::Arc, thread::available_parallelism};
+use std::sync::Arc;
 
-use litellm_host_python::{release_gil, run_async};
+#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
+use std::{num::NonZero, thread::available_parallelism};
+
+#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
+use litellm_host_python::release_gil;
+use litellm_host_python::run_async;
 use litellm_token_counter::{
     CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
 };
@@ -28,17 +33,66 @@ pub(crate) struct TokenCounter {
 impl TokenCounter {
     #[new]
     fn new(py: Python<'_>, tokenizer_json: &str) -> PyResult<Self> {
-        Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
+        #[cfg(feature = "fast")]
+        {
+            Self::load(py, || CoreTokenCounter::from_json_fast(tokenizer_json))
+        }
+        #[cfg(all(not(feature = "fast"), feature = "huggingface"))]
+        {
+            Self::load(py, || CoreTokenCounter::from_json(tokenizer_json))
+        }
+        #[cfg(not(any(feature = "fast", feature = "huggingface")))]
+        {
+            let _ = (py, tokenizer_json);
+            Err(RustBridgeDeclined::new_err(
+                "tokenizer backend requires the fast or huggingface feature",
+            ))
+        }
     }
 
     #[staticmethod]
     fn from_cl100k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
-        Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
+        #[cfg(feature = "fast")]
+        {
+            Self::load(py, || CoreTokenCounter::from_cl100k_ranks(rank_file))
+        }
+        #[cfg(not(feature = "fast"))]
+        {
+            let _ = (py, rank_file);
+            Err(RustBridgeDeclined::new_err(
+                "tokenizer backend requires the fast feature",
+            ))
+        }
     }
 
     #[staticmethod]
     fn from_o200k_ranks(py: Python<'_>, rank_file: &str) -> PyResult<Self> {
-        Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
+        #[cfg(feature = "fast")]
+        {
+            Self::load(py, || CoreTokenCounter::from_o200k_ranks(rank_file))
+        }
+        #[cfg(not(feature = "fast"))]
+        {
+            let _ = (py, rank_file);
+            Err(RustBridgeDeclined::new_err(
+                "tokenizer backend requires the fast feature",
+            ))
+        }
+    }
+
+    #[staticmethod]
+    fn from_tiktoken(py: Python<'_>, encoding: &str) -> PyResult<Self> {
+        #[cfg(feature = "tiktoken")]
+        {
+            Self::load(py, || CoreTokenCounter::from_tiktoken(encoding))
+        }
+        #[cfg(not(feature = "tiktoken"))]
+        {
+            let _ = (py, encoding);
+            Err(RustBridgeDeclined::new_err(
+                "tokenizer backend requires the tiktoken feature",
+            ))
+        }
     }
 
     fn acount_request<'py>(&self, py: Python<'py>, body: &[u8]) -> PyResult<Bound<'py, PyAny>> {
@@ -62,6 +116,7 @@ impl TokenCounter {
 }
 
 impl TokenCounter {
+    #[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
     fn load(
         py: Python<'_>,
         load: impl FnOnce() -> Result<CoreTokenCounter, Error> + Send,
@@ -74,6 +129,7 @@ impl TokenCounter {
     }
 }
 
+#[cfg(any(feature = "fast", feature = "huggingface", feature = "tiktoken"))]
 fn encode_parallelism() -> usize {
     available_parallelism().map_or(1, NonZero::get)
 }
@@ -86,7 +142,10 @@ fn count_body(counter: &CoreTokenCounter, body: &[u8]) -> Result<InputTokenCount
 fn token_count_error_to_pyerr(error: Error) -> PyErr {
     let message = error.to_string();
     match error {
-        Error::Load(_) | Error::Ranks(_) | Error::UnicodeClasses => PyValueError::new_err(message),
+        Error::Load(_)
+        | Error::Ranks(_)
+        | Error::UnicodeClasses
+        | Error::UnsupportedTokenizer(_) => PyValueError::new_err(message),
         Error::RequestParse(_)
         | Error::MissingInput
         | Error::FloatText

litellm-rust/crates/secrets-aws/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "litellm-secrets-aws"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-aws.workspace = true
+litellm-secrets-types.workspace = true
+litellm-core-utils.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+tracing = "0.1"
+veil.workspace = true
+aws-sdk-kms = "1.120.0"
+aws-sdk-secretsmanager = "1.117.0"
+aws-credential-types = "1.3.0"
+
+[dev-dependencies]
+base64.workspace = true
+rstest.workspace = true
+tokio.workspace = true
+wiremock = "0.6.5"

litellm-rust/crates/secrets-aws/src/auth.rs

@@ -0,0 +1,79 @@
+use std::sync::Arc;
+
+use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
+use litellm_auth_aws::{
+    AwsAuthConfig,
+    constants::{AWS_DEFAULT_REGION, AWS_REGION, AWS_REGION_NAME},
+    resolve_credentials,
+};
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::KeyManagementSettings;
+
+use crate::Error;
+
+#[derive(Clone)]
+pub(crate) struct Credentials {
+    config: AwsAuthConfig,
+    environment: Arc<dyn Lookup + Send + Sync>,
+}
+
+impl Credentials {
+    pub(crate) fn new(
+        settings: &KeyManagementSettings,
+        environment: Arc<dyn Lookup + Send + Sync>,
+    ) -> Self {
+        Self {
+            config: AwsAuthConfig {
+                region_name: region(settings, environment.as_ref()).ok(),
+                role_name: settings.aws_role_name.clone(),
+                session_name: settings.aws_session_name.clone(),
+                external_id: settings
+                    .aws_external_id
+                    .as_ref()
+                    .map(|v| v.expose().to_owned()),
+                profile_name: settings.aws_profile_name.clone(),
+                web_identity_token: settings
+                    .aws_web_identity_token
+                    .as_ref()
+                    .map(|v| v.expose().to_owned()),
+                sts_endpoint: settings.aws_sts_endpoint.clone(),
+                ..Default::default()
+            },
+            environment,
+        }
+    }
+}
+
+impl ProvideCredentials for Credentials {
+    fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
+    where
+        Self: 'a,
+    {
+        future::ProvideCredentials::new(async {
+            resolve_credentials(self.config.clone(), &|name| self.environment.get(name))
+                .await
+                .map_err(|_| {
+                    CredentialsError::provider_error("secret manager authentication failed")
+                })
+        })
+    }
+}
+
+pub(crate) fn region(
+    settings: &KeyManagementSettings,
+    environment: &dyn Lookup,
+) -> Result<String, Error> {
+    settings
+        .aws_region_name
+        .clone()
+        .or_else(|| environment.get(AWS_REGION_NAME))
+        .or_else(|| environment.get(AWS_REGION))
+        .or_else(|| environment.get(AWS_DEFAULT_REGION))
+        .ok_or(Error::MissingRegion)
+}
+
+impl std::fmt::Debug for Credentials {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Credentials").finish_non_exhaustive()
+    }
+}

litellm-rust/crates/secrets-aws/src/error.rs

@@ -0,0 +1,31 @@
+use aws_sdk_secretsmanager::error::SdkError;
+
+#[derive(thiserror::Error, veil::Redact)]
+pub enum Error {
+    #[error("AWS authentication failed")]
+    Auth(#[from] #[redact] litellm_auth_aws::Error),
+    #[error("AWS region is not configured")]
+    MissingRegion,
+    #[error("KMS response has no plaintext")]
+    MissingPlaintext,
+    #[error("AWS request timed out")]
+    Timeout,
+    #[error("AWS KMS decrypt failed")]
+    Decrypt(#[from] #[redact] Box<SdkError<aws_sdk_kms::operation::decrypt::DecryptError>>),
+    #[error("AWS Secrets Manager read failed")]
+    Read(#[from] #[redact] Box<SdkError<aws_sdk_secretsmanager::operation::get_secret_value::GetSecretValueError>>),
+    #[error("AWS Secrets Manager create failed")]
+    Create(#[from] #[redact] Box<SdkError<aws_sdk_secretsmanager::operation::create_secret::CreateSecretError>>),
+    #[error("AWS Secrets Manager update failed")]
+    Put(#[from] #[redact] Box<SdkError<aws_sdk_secretsmanager::operation::put_secret_value::PutSecretValueError>>),
+    #[error("AWS Secrets Manager delete failed")]
+    Delete(#[from] #[redact] Box<SdkError<aws_sdk_secretsmanager::operation::delete_secret::DeleteSecretError>>),
+    #[error("AWS Secrets Manager replication failed")]
+    Replicate(#[from] #[redact] Box<SdkError<aws_sdk_secretsmanager::operation::replicate_secret_to_regions::ReplicateSecretToRegionsError>>),
+    #[error("AWS Secrets Manager response has no string payload")]
+    MissingString,
+    #[error("primary secret is not a JSON object")]
+    PrimarySecret,
+    #[error(transparent)]
+    Operation(#[from] litellm_secrets_types::Error),
+}

litellm-rust/crates/secrets-aws/src/kms.rs

@@ -0,0 +1,63 @@
+use litellm_auth_aws::constants::AWS_REGION_NAME;
+use std::sync::Arc;
+
+use aws_sdk_kms::{
+    Client,
+    config::{BehaviorVersion, Region},
+    primitives::Blob,
+};
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::KeyManagementSettings;
+
+use crate::{Error, auth};
+
+#[derive(Clone)]
+pub struct AwsKms {
+    client: Client,
+}
+
+impl AwsKms {
+    pub fn new(client: Client) -> Self {
+        Self { client }
+    }
+
+    pub async fn decrypt(&self, ciphertext: Vec<u8>) -> Result<Vec<u8>, Error> {
+        let response = self
+            .client
+            .decrypt()
+            .ciphertext_blob(Blob::new(ciphertext))
+            .send()
+            .await
+            .map_err(|error| Error::Decrypt(Box::new(error)))?;
+        Ok(response
+            .plaintext
+            .ok_or(Error::MissingPlaintext)?
+            .into_inner())
+    }
+}
+
+pub fn validate_environment(environment: &dyn Lookup) -> Result<(), Error> {
+    environment
+        .get(AWS_REGION_NAME)
+        .map(|_| ())
+        .ok_or(Error::MissingRegion)
+}
+
+pub fn load_aws_kms(
+    use_aws_kms: Option<bool>,
+    settings: &KeyManagementSettings,
+    environment: Arc<dyn Lookup + Send + Sync>,
+) -> Result<Option<AwsKms>, Error> {
+    if use_aws_kms != Some(true) {
+        return Ok(None);
+    }
+    if settings.aws_region_name.is_none() {
+        validate_environment(environment.as_ref())?;
+    }
+    let config = aws_sdk_kms::Config::builder()
+        .behavior_version(BehaviorVersion::latest())
+        .region(Region::new(auth::region(settings, environment.as_ref())?))
+        .credentials_provider(auth::Credentials::new(settings, environment))
+        .build();
+    Ok(Some(AwsKms::new(Client::from_conf(config))))
+}

litellm-rust/crates/secrets-aws/src/lib.rs

@@ -0,0 +1,10 @@
+#![forbid(unsafe_code)]
+
+mod auth;
+mod error;
+pub mod kms;
+pub mod secret_manager;
+
+pub use error::Error;
+pub use kms::{AwsKms, load_aws_kms};
+pub use secret_manager::{AwsSecretWriteSettings, AwsSecretsManagerV2, RotationResponse};

litellm-rust/crates/secrets-aws/src/secret_manager.rs

@@ -0,0 +1,287 @@
+use litellm_auth_aws::constants::AWS_BEDROCK_RUNTIME_ENDPOINT;
+use std::{collections::BTreeMap, sync::Arc};
+
+use aws_sdk_secretsmanager::{
+    Client,
+    config::{BehaviorVersion, Region},
+    operation::{
+        create_secret::CreateSecretOutput, delete_secret::DeleteSecretOutput,
+        put_secret_value::PutSecretValueOutput,
+        replicate_secret_to_regions::ReplicateSecretToRegionsOutput,
+    },
+    types::{ReplicaRegionType, Tag},
+};
+use litellm_auth_aws::constants::{
+    AWS_ACCESS_KEY_ID, AWS_REGION, AWS_REGION_NAME, AWS_SECRET_ACCESS_KEY,
+};
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::{
+    BaseSecretManager, KeyManagementSettings, Secret, SecretValue, async_rotate_secret,
+};
+use serde_json::Value;
+
+use crate::{Error, auth};
+
+#[derive(Clone)]
+pub struct AwsSecretsManagerV2 {
+    client: Client,
+    write_settings: AwsSecretWriteSettings,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct AwsSecretWriteSettings {
+    pub kms_key_id: Option<String>,
+    pub tags: Option<BTreeMap<String, String>>,
+    pub replica_regions: Option<Vec<String>>,
+}
+
+impl From<&KeyManagementSettings> for AwsSecretWriteSettings {
+    fn from(settings: &KeyManagementSettings) -> Self {
+        Self {
+            kms_key_id: settings.kms_key_id.clone(),
+            tags: settings.tags.clone(),
+            replica_regions: settings.replica_regions.clone(),
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum RotationResponse {
+    Created(CreateSecretOutput),
+    Updated(PutSecretValueOutput),
+}
+
+impl AwsSecretsManagerV2 {
+    pub fn new(client: Client, write_settings: AwsSecretWriteSettings) -> Self {
+        Self {
+            client,
+            write_settings,
+        }
+    }
+
+    pub fn load_aws_secret_manager(
+        use_aws_secret_manager: Option<bool>,
+        settings: KeyManagementSettings,
+        environment: Arc<dyn Lookup + Send + Sync>,
+    ) -> Result<Option<Self>, Error> {
+        if use_aws_secret_manager != Some(true) {
+            return Ok(None);
+        }
+        let builder = aws_sdk_secretsmanager::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new(auth::region(&settings, environment.as_ref())?))
+            .credentials_provider(auth::Credentials::new(&settings, environment.clone()));
+        let config = match environment.get(AWS_BEDROCK_RUNTIME_ENDPOINT) {
+            Some(url) => builder
+                .endpoint_url(url.replace("bedrock-runtime", "secretsmanager"))
+                .build(),
+            None => builder.build(),
+        };
+        Ok(Some(Self::new(
+            Client::from_conf(config),
+            (&settings).into(),
+        )))
+    }
+
+    pub async fn read_secret_for_resolver(
+        &self,
+        name: &str,
+        primary_name: Option<&str>,
+        environment: &(dyn Lookup + Sync),
+    ) -> Result<Option<Secret>, Error> {
+        if bootstrap_key(name) {
+            return Ok(environment
+                .get(name)
+                .map(SecretValue::new)
+                .map(Secret::String));
+        }
+        match primary_name.filter(|name| !name.is_empty()) {
+            None => self
+                .async_read_secret(name)
+                .await
+                .map(|value| value.map(Secret::String)),
+            Some(primary) => {
+                let value = if bootstrap_key(primary) {
+                    environment.get(primary).map(SecretValue::new)
+                } else {
+                    self.async_read_secret(primary).await?
+                };
+                let Some(value) = value else {
+                    return Ok(None);
+                };
+                let object: Value =
+                    serde_json::from_str(value.expose()).map_err(|_| Error::PrimarySecret)?;
+                let object = object.as_object().ok_or(Error::PrimarySecret)?;
+                Ok(object.get(name).cloned().map(Secret::from_json))
+            }
+        }
+    }
+
+    pub async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
+        match self.client.get_secret_value().secret_id(name).send().await {
+            Ok(response) => response
+                .secret_string
+                .map(SecretValue::new)
+                .map(Some)
+                .ok_or(Error::MissingString),
+            Err(error)
+                if matches!(
+                    &error,
+                    aws_sdk_secretsmanager::error::SdkError::TimeoutError(_)
+                ) || matches!(&error, aws_sdk_secretsmanager::error::SdkError::DispatchFailure(failure) if failure.is_timeout()) =>
+            {
+                Err(Error::Timeout)
+            }
+            Err(error)
+                if error
+                    .as_service_error()
+                    .is_some_and(|error| error.is_resource_not_found_exception()) =>
+            {
+                Ok(None)
+            }
+            Err(error) => Err(Error::Read(Box::new(error))),
+        }
+    }
+
+    pub async fn async_write_secret(
+        &self,
+        name: &str,
+        value: &SecretValue,
+        description: Option<&str>,
+    ) -> Result<CreateSecretOutput, Error> {
+        let response = self
+            .client
+            .create_secret()
+            .name(name)
+            .secret_string(value.expose())
+            .set_description(description.filter(|v| !v.is_empty()).map(str::to_owned))
+            .set_kms_key_id(
+                self.write_settings
+                    .kms_key_id
+                    .clone()
+                    .filter(|v| !v.is_empty()),
+            )
+            .set_tags(self.write_settings.tags.as_ref().map(|tags| {
+                tags.iter()
+                    .map(|(key, value)| Tag::builder().key(key).value(value).build())
+                    .collect()
+            }))
+            .send()
+            .await
+            .map_err(|error| Error::Create(Box::new(error)))?;
+        if let Some(regions) = &self.write_settings.replica_regions
+            && !regions.is_empty()
+            && self.async_replicate_secret(name, regions).await.is_err()
+        {
+            tracing::warn!("secret created but replication failed");
+        }
+        Ok(response)
+    }
+
+    pub async fn async_replicate_secret(
+        &self,
+        name: &str,
+        regions: &[String],
+    ) -> Result<Option<ReplicateSecretToRegionsOutput>, Error> {
+        if regions.is_empty() {
+            return Ok(None);
+        }
+        self.client
+            .replicate_secret_to_regions()
+            .secret_id(name)
+            .set_add_replica_regions(Some(
+                regions
+                    .iter()
+                    .map(|region| ReplicaRegionType::builder().region(region).build())
+                    .collect(),
+            ))
+            .send()
+            .await
+            .map(Some)
+            .map_err(|error| Error::Replicate(Box::new(error)))
+    }
+
+    pub async fn async_put_secret_value(
+        &self,
+        name: &str,
+        value: &SecretValue,
+    ) -> Result<PutSecretValueOutput, Error> {
+        self.client
+            .put_secret_value()
+            .secret_id(name)
+            .secret_string(value.expose())
+            .send()
+            .await
+            .map_err(|error| Error::Put(Box::new(error)))
+    }
+
+    pub async fn async_delete_secret(
+        &self,
+        name: &str,
+        recovery_window_in_days: i64,
+    ) -> Result<DeleteSecretOutput, Error> {
+        self.client
+            .delete_secret()
+            .secret_id(name)
+            .recovery_window_in_days(recovery_window_in_days)
+            .send()
+            .await
+            .map_err(|error| Error::Delete(Box::new(error)))
+    }
+
+    pub async fn async_rotate_secret(
+        &self,
+        current_name: &str,
+        new_name: &str,
+        value: &SecretValue,
+    ) -> Result<RotationResponse, Error> {
+        if current_name == new_name {
+            return self
+                .async_put_secret_value(current_name, value)
+                .await
+                .map(RotationResponse::Updated);
+        }
+        async_rotate_secret(self, current_name, new_name, value)
+            .await
+            .map(RotationResponse::Created)
+    }
+}
+
+impl BaseSecretManager for AwsSecretsManagerV2 {
+    type Error = Error;
+    type WriteResponse = CreateSecretOutput;
+    type DeleteResponse = DeleteSecretOutput;
+
+    async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
+        self.async_read_secret(name).await
+    }
+
+    async fn async_write_secret(
+        &self,
+        name: &str,
+        value: &SecretValue,
+        description: Option<&str>,
+    ) -> Result<CreateSecretOutput, Error> {
+        self.async_write_secret(name, value, description).await
+    }
+
+    async fn async_delete_secret(
+        &self,
+        name: &str,
+        recovery_window_in_days: i64,
+    ) -> Result<DeleteSecretOutput, Error> {
+        self.async_delete_secret(name, recovery_window_in_days)
+            .await
+    }
+}
+
+fn bootstrap_key(name: &str) -> bool {
+    matches!(
+        name,
+        AWS_ACCESS_KEY_ID
+            | AWS_SECRET_ACCESS_KEY
+            | AWS_REGION_NAME
+            | AWS_REGION
+            | AWS_BEDROCK_RUNTIME_ENDPOINT
+    )
+}

litellm-rust/crates/secrets-aws/tests/kms.rs

@@ -0,0 +1,59 @@
+use aws_sdk_kms::{
+    Client,
+    config::{BehaviorVersion, Credentials, Region, retry::RetryConfig},
+};
+use base64::{Engine, engine::general_purpose::STANDARD};
+use litellm_secrets_aws::AwsKms;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{body_json, header},
+};
+
+#[tokio::test]
+async fn kms_decrypt_calls_the_sdk_without_applying_lookup_policy() {
+    let server = MockServer::start().await;
+    let plaintext = "  private-value\n";
+    Mock::given(header("x-amz-target", "TrentService.Decrypt"))
+        .and(body_json(
+            serde_json::json!({"CiphertextBlob": STANDARD.encode("encrypted")}),
+        ))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .set_body_json(serde_json::json!({"Plaintext": STANDARD.encode(plaintext)})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let client = Client::from_conf(
+        aws_sdk_kms::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new("us-east-1"))
+            .credentials_provider(Credentials::new("test", "test", None, None, "test"))
+            .endpoint_url(server.uri())
+            .retry_config(RetryConfig::disabled())
+            .build(),
+    );
+    let manager = AwsKms::new(client);
+    assert_eq!(
+        manager.decrypt(b"encrypted".to_vec()).await.unwrap(),
+        plaintext.as_bytes()
+    );
+}
+
+#[test]
+fn disabled_kms_loader_does_not_require_environment_configuration() {
+    use litellm_secrets_aws::load_aws_kms;
+    use litellm_secrets_types::KeyManagementSettings;
+    use std::sync::Arc;
+    for enabled in [None, Some(false)] {
+        assert!(
+            load_aws_kms(
+                enabled,
+                &KeyManagementSettings::default(),
+                Arc::new(|_: &str| None)
+            )
+            .unwrap()
+            .is_none()
+        );
+    }
+}

litellm-rust/crates/secrets-aws/tests/secret_manager.rs

@@ -0,0 +1,312 @@
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use aws_sdk_secretsmanager::{
+    Client,
+    config::{BehaviorVersion, Credentials, Region, retry::RetryConfig},
+};
+use litellm_secrets_aws::{AwsSecretsManagerV2, Error, RotationResponse};
+use litellm_secrets_types::{KeyManagementSettings, SecretValue};
+use serde_json::json;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{body_partial_json, header},
+};
+
+fn manager(server: &MockServer, settings: KeyManagementSettings) -> AwsSecretsManagerV2 {
+    let client = Client::from_conf(
+        aws_sdk_secretsmanager::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new("us-east-1"))
+            .credentials_provider(Credentials::new("test", "test", None, None, "test"))
+            .endpoint_url(server.uri())
+            .retry_config(RetryConfig::disabled())
+            .build(),
+    );
+    AwsSecretsManagerV2::new(client, (&settings).into())
+}
+
+#[rstest::rstest]
+#[case::string_value("KEY", Some("value"))]
+#[case::missing_value("missing", None)]
+#[case::non_string_value("BOOL", None)]
+#[tokio::test]
+async fn primary_lookup_preserves_read_semantics(
+    #[case] name: &str,
+    #[case] expected: Option<&str>,
+) {
+    let server = MockServer::start().await;
+    Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
+        .and(body_partial_json(json!({"SecretId":"primary"})))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(
+                json!({"SecretString":json!({"KEY":"value", "BOOL":true}).to_string()}),
+            ),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let manager = manager(&server, KeyManagementSettings::default());
+    assert_eq!(
+        manager
+            .read_secret_for_resolver(name, Some("primary"), &|_: &str| None)
+            .await
+            .unwrap()
+            .and_then(|v| v.as_str().map(str::to_owned))
+            .as_deref(),
+        expected
+    );
+}
+
+#[rstest::rstest]
+#[case::access_key("AWS_ACCESS_KEY_ID")]
+#[case::secret_access_key("AWS_SECRET_ACCESS_KEY")]
+#[case::region_name("AWS_REGION_NAME")]
+#[case::region("AWS_REGION")]
+#[case::bedrock_endpoint("AWS_BEDROCK_RUNTIME_ENDPOINT")]
+#[tokio::test]
+async fn bootstrap_keys_bypass_primary_lookup(#[case] name: &str) {
+    let server = MockServer::start().await;
+    let manager = manager(&server, KeyManagementSettings::default());
+    assert_eq!(
+        manager
+            .read_secret_for_resolver(name, Some("primary"), &|_: &str| Some("bootstrap".into()))
+            .await
+            .unwrap()
+            .unwrap()
+            .as_str()
+            .unwrap(),
+        "bootstrap"
+    );
+}
+
+#[tokio::test]
+async fn failed_read_returns_none_but_invalid_primary_json_is_an_error() {
+    let server = MockServer::start().await;
+    Mock::given(body_partial_json(json!({"SecretId":"missing"})))
+        .respond_with(
+            ResponseTemplate::new(400).set_body_json(json!({"__type":"ResourceNotFoundException"})),
+        )
+        .mount(&server)
+        .await;
+    Mock::given(body_partial_json(json!({"SecretId":"invalid"})))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"SecretString":"not-json"})))
+        .mount(&server)
+        .await;
+    let manager = manager(&server, KeyManagementSettings::default());
+    assert!(
+        manager
+            .async_read_secret("missing")
+            .await
+            .unwrap()
+            .is_none()
+    );
+    assert!(matches!(
+        manager
+            .read_secret_for_resolver("KEY", Some("invalid"), &|_: &str| None)
+            .await,
+        Err(Error::PrimarySecret)
+    ));
+}
+
+#[tokio::test]
+async fn same_name_rotation_uses_put_and_returns_its_response() {
+    let server = MockServer::start().await;
+    Mock::given(header("x-amz-target", "secretsmanager.PutSecretValue"))
+        .and(body_partial_json(
+            json!({"SecretId":"key", "SecretString":"replacement"}),
+        ))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(json!({"Name":"key", "VersionId":"version"})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let response = manager(&server, KeyManagementSettings::default())
+        .async_rotate_secret("key", "key", &SecretValue::new("replacement"))
+        .await
+        .unwrap();
+    match response {
+        RotationResponse::Updated(output) => assert_eq!(output.version_id(), Some("version")),
+        _ => panic!("rotation created a second secret"),
+    }
+    assert_eq!(server.received_requests().await.unwrap().len(), 1);
+}
+
+#[tokio::test]
+async fn renamed_rotation_reads_creates_verifies_then_deletes() {
+    let server = MockServer::start().await;
+    let step = AtomicUsize::new(0);
+    Mock::given(wiremock::matchers::method("POST"))
+        .respond_with(move |request: &wiremock::Request| {
+            let body: serde_json::Value = request.body_json().unwrap();
+            let action = request
+                .headers
+                .get("x-amz-target")
+                .unwrap()
+                .to_str()
+                .unwrap();
+            match step.fetch_add(1, Ordering::SeqCst) {
+                0 => {
+                    assert_eq!(action, "secretsmanager.GetSecretValue");
+                    assert_eq!(body["SecretId"], "old");
+                    ResponseTemplate::new(200).set_body_json(json!({"SecretString":"old-value"}))
+                }
+                1 => {
+                    assert_eq!(action, "secretsmanager.CreateSecret");
+                    assert_eq!(body["Name"], "new");
+                    assert_eq!(body["Description"], "Rotated from old");
+                    assert_eq!(body["SecretString"], "replacement");
+                    ResponseTemplate::new(200).set_body_json(json!({"Name":"new"}))
+                }
+                2 => {
+                    assert_eq!(action, "secretsmanager.GetSecretValue");
+                    assert_eq!(body["SecretId"], "new");
+                    ResponseTemplate::new(200).set_body_json(json!({"SecretString":"replacement"}))
+                }
+                3 => {
+                    assert_eq!(action, "secretsmanager.DeleteSecret");
+                    assert_eq!(body["SecretId"], "old");
+                    assert_eq!(body["RecoveryWindowInDays"], 7);
+                    ResponseTemplate::new(200).set_body_json(json!({"Name":"old"}))
+                }
+                _ => panic!("unexpected request"),
+            }
+        })
+        .expect(4)
+        .mount(&server)
+        .await;
+    assert!(matches!(
+        manager(&server, KeyManagementSettings::default())
+            .async_rotate_secret("old", "new", &SecretValue::new("replacement"))
+            .await
+            .unwrap(),
+        RotationResponse::Created(_)
+    ));
+}
+
+#[tokio::test]
+async fn creation_passes_tags_and_kms_and_survives_replication_failure() {
+    let server = MockServer::start().await;
+    Mock::given(header("x-amz-target", "secretsmanager.CreateSecret"))
+        .and(body_partial_json(json!({"Name":"key", "SecretString":"value", "KmsKeyId":"kms-key", "Tags":[{"Key":"stage", "Value":"test"}]})))
+        .respond_with(ResponseTemplate::new(200).set_body_json(json!({"Name":"key"}))).expect(1).mount(&server).await;
+    Mock::given(header(
+        "x-amz-target",
+        "secretsmanager.ReplicateSecretToRegions",
+    ))
+    .and(body_partial_json(
+        json!({"SecretId":"key", "AddReplicaRegions":[{"Region":"replica-region"}]}),
+    ))
+    .respond_with(
+        ResponseTemplate::new(400).set_body_json(json!({"__type":"InvalidRequestException"})),
+    )
+    .expect(1)
+    .mount(&server)
+    .await;
+    let settings = KeyManagementSettings {
+        kms_key_id: Some("kms-key".into()),
+        tags: Some(std::collections::BTreeMap::from([(
+            "stage".into(),
+            "test".into(),
+        )])),
+        replica_regions: Some(vec!["replica-region".into()]),
+        ..Default::default()
+    };
+    let manager = manager(&server, settings);
+    assert_eq!(
+        manager
+            .async_write_secret("key", &SecretValue::new("value"), None)
+            .await
+            .unwrap()
+            .name(),
+        Some("key")
+    );
+    assert!(
+        manager
+            .async_replicate_secret("key", &[])
+            .await
+            .unwrap()
+            .is_none()
+    );
+}
+
+#[tokio::test]
+async fn credential_failures_are_not_swallowed_as_missing_secrets() {
+    use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
+    #[derive(Debug)]
+    struct FailedCredentials;
+    impl ProvideCredentials for FailedCredentials {
+        fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
+        where
+            Self: 'a,
+        {
+            future::ProvideCredentials::ready(Err(CredentialsError::provider_error(
+                "private-auth-detail",
+            )))
+        }
+    }
+    let server = MockServer::start().await;
+    let config = aws_sdk_secretsmanager::Config::builder()
+        .behavior_version(BehaviorVersion::latest())
+        .region(Region::new("us-east-1"))
+        .credentials_provider(FailedCredentials)
+        .endpoint_url(server.uri())
+        .retry_config(RetryConfig::disabled())
+        .build();
+    let manager = AwsSecretsManagerV2::new(Client::from_conf(config), Default::default());
+    let error = manager.async_read_secret("key").await.unwrap_err();
+    assert!(!format!("{error:?}").contains("private-auth-detail"));
+    assert!(matches!(error, Error::Read(_)));
+    assert!(server.received_requests().await.unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn read_timeout_is_an_error_and_cannot_be_mistaken_for_missing() {
+    use std::time::Duration;
+    let server = MockServer::start().await;
+    Mock::given(wiremock::matchers::method("POST"))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .set_delay(Duration::from_secs(1))
+                .set_body_json(json!({"SecretString":"late"})),
+        )
+        .mount(&server)
+        .await;
+    let config = aws_sdk_secretsmanager::Config::builder()
+        .behavior_version(BehaviorVersion::latest())
+        .region(Region::new("us-east-1"))
+        .credentials_provider(Credentials::new("test", "test", None, None, "test"))
+        .endpoint_url(server.uri())
+        .retry_config(RetryConfig::disabled())
+        .timeout_config(
+            aws_sdk_secretsmanager::config::timeout::TimeoutConfig::builder()
+                .operation_timeout(Duration::from_millis(30))
+                .build(),
+        )
+        .build();
+    let manager = AwsSecretsManagerV2::new(Client::from_conf(config), Default::default());
+    assert!(matches!(
+        manager.async_read_secret("key").await,
+        Err(Error::Timeout)
+    ));
+}
+
+#[rstest::rstest]
+#[case::denied(400, "AccessDeniedException")]
+#[case::throttled(400, "ThrottlingException")]
+#[case::unavailable(503, "ServiceUnavailableException")]
+#[tokio::test]
+async fn service_failures_remain_errors(#[case] status: u16, #[case] code: &str) {
+    let server = MockServer::start().await;
+    Mock::given(header("x-amz-target", "secretsmanager.GetSecretValue"))
+        .respond_with(ResponseTemplate::new(status).set_body_json(json!({"__type":code})))
+        .expect(1)
+        .mount(&server)
+        .await;
+    assert!(matches!(
+        manager(&server, KeyManagementSettings::default())
+            .async_read_secret("key")
+            .await,
+        Err(Error::Read(_))
+    ));
+}

litellm-rust/crates/secrets-google/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "litellm-secrets-google"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-gcp = { workspace = true, features = ["google-sdk"] }
+litellm-secrets-types.workspace = true
+litellm-auth-types.workspace = true
+litellm-core-utils.workspace = true
+base64.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+moka.workspace = true
+veil.workspace = true
+google-cloud-kms-v1 = "1.14.0"
+google-cloud-gax = { version = "1.14.0", default-features = false }
+percent-encoding = "2.3"
+serde.workspace = true
+reqwest.workspace = true
+
+[dev-dependencies]
+google-cloud-auth.workspace = true
+rstest.workspace = true
+tokio.workspace = true
+wiremock = "0.6.5"

litellm-rust/crates/secrets-google/src/auth.rs

@@ -0,0 +1,21 @@
+use std::sync::Arc;
+
+use litellm_auth_gcp::{GoogleCredentials, VertexConfig};
+use litellm_auth_types::{InputSource, Sourced};
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::SecretValue;
+
+pub(crate) fn credentials(
+    project: Option<String>,
+    credentials: Option<SecretValue>,
+    environment: Arc<dyn Lookup + Send + Sync>,
+) -> GoogleCredentials {
+    GoogleCredentials::new(
+        VertexConfig::new(
+            credentials.map(|value| Sourced::new(value, InputSource::Environment)),
+            project,
+            None,
+        ),
+        Arc::new(move |name| environment.get(name)),
+    )
+}

litellm-rust/crates/secrets-google/src/error.rs

@@ -0,0 +1,43 @@
+#[derive(thiserror::Error, veil::Redact)]
+pub enum Error {
+    #[error("Google KMS client configuration failed")]
+    Client(
+        #[from]
+        #[redact]
+        google_cloud_gax::client_builder::Error,
+    ),
+    #[error("Google authentication failed")]
+    Auth(
+        #[from]
+        #[redact]
+        litellm_auth_types::Error,
+    ),
+    #[error("Google KMS request failed")]
+    Kms(
+        #[from]
+        #[redact]
+        google_cloud_gax::error::Error,
+    ),
+    #[error("Google Secret Manager HTTP request failed")]
+    Http(
+        #[from]
+        #[redact]
+        reqwest::Error,
+    ),
+    #[error("Google Secret Manager returned HTTP {0}")]
+    Status(u16),
+    #[error("Google Secret Manager returned no payload")]
+    MissingPayload,
+    #[error("required environment variable is missing: {0}")]
+    MissingEnvironment(&'static str),
+    #[error("invalid refresh interval")]
+    RefreshInterval,
+    #[error("payload is not valid base64")]
+    Base64(#[from] base64::DecodeError),
+    #[error("decrypted value is not UTF-8")]
+    Utf8,
+    #[error("invalid Google Secret Manager endpoint")]
+    Endpoint,
+    #[error("Google Secret Manager requires an enterprise license")]
+    EnterpriseRequired,
+}

litellm-rust/crates/secrets-google/src/kms.rs

@@ -0,0 +1,67 @@
+use std::sync::Arc;
+
+use google_cloud_kms_v1::client::KeyManagementService;
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::SecretValue;
+
+use crate::{Error, auth};
+
+const GOOGLE_APPLICATION_CREDENTIALS: &str = "GOOGLE_APPLICATION_CREDENTIALS";
+const GOOGLE_KMS_RESOURCE_NAME: &str = "GOOGLE_KMS_RESOURCE_NAME";
+
+#[derive(Clone)]
+pub struct GoogleKms {
+    client: KeyManagementService,
+    resource_name: String,
+}
+
+impl GoogleKms {
+    pub fn new(client: KeyManagementService, resource_name: String) -> Self {
+        Self {
+            client,
+            resource_name,
+        }
+    }
+
+    pub async fn decrypt(&self, ciphertext: Vec<u8>) -> Result<Vec<u8>, Error> {
+        let response = self
+            .client
+            .decrypt()
+            .set_name(&self.resource_name)
+            .set_ciphertext(ciphertext)
+            .send()
+            .await?;
+        Ok(response.plaintext.to_vec())
+    }
+}
+
+pub fn validate_environment(environment: &dyn Lookup) -> Result<(), Error> {
+    for key in [GOOGLE_APPLICATION_CREDENTIALS, GOOGLE_KMS_RESOURCE_NAME] {
+        if environment.get(key).is_none() {
+            return Err(Error::MissingEnvironment(key));
+        }
+    }
+    Ok(())
+}
+
+pub async fn load_google_kms(
+    use_google_kms: Option<bool>,
+    environment: Arc<dyn Lookup + Send + Sync>,
+) -> Result<Option<GoogleKms>, Error> {
+    if use_google_kms != Some(true) {
+        return Ok(None);
+    }
+    validate_environment(environment.as_ref())?;
+    let credentials = environment
+        .get(GOOGLE_APPLICATION_CREDENTIALS)
+        .ok_or(Error::MissingEnvironment(GOOGLE_APPLICATION_CREDENTIALS))?;
+    let resource_name = environment
+        .get(GOOGLE_KMS_RESOURCE_NAME)
+        .ok_or(Error::MissingEnvironment(GOOGLE_KMS_RESOURCE_NAME))?;
+    let credentials = auth::credentials(None, Some(SecretValue::new(credentials)), environment);
+    let client = KeyManagementService::builder()
+        .with_credentials(credentials)
+        .build()
+        .await?;
+    Ok(Some(GoogleKms::new(client, resource_name)))
+}

litellm-rust/crates/secrets-google/src/lib.rs

@@ -0,0 +1,10 @@
+#![forbid(unsafe_code)]
+
+mod auth;
+mod error;
+pub mod kms;
+pub mod secret_manager;
+
+pub use error::Error;
+pub use kms::{GoogleKms, load_google_kms};
+pub use secret_manager::GoogleSecretManager;

litellm-rust/crates/secrets-google/src/secret_manager.rs

@@ -0,0 +1,157 @@
+use std::{sync::Arc, time::Duration};
+
+use base64::{Engine, engine::general_purpose::STANDARD};
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets_types::{Secret, SecretValue};
+use moka::future::Cache;
+use serde::Deserialize;
+
+use litellm_auth_gcp::GoogleCredentials;
+
+use crate::{Error, auth};
+
+const GOOGLE_SECRET_MANAGER_PROJECT_ID: &str = "GOOGLE_SECRET_MANAGER_PROJECT_ID";
+const GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL: &str = "GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL";
+const SECRET_MANAGER_REFRESH_INTERVAL: &str = "SECRET_MANAGER_REFRESH_INTERVAL";
+const GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER: &str =
+    "GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER";
+const GCS_PATH_SERVICE_ACCOUNT: &str = "GCS_PATH_SERVICE_ACCOUNT";
+const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(86400);
+const DEFAULT_CACHE_TTL: Duration = Duration::from_secs(600);
+const CACHE_CAPACITY: u64 = 200;
+
+#[derive(Clone)]
+pub struct GoogleSecretManager {
+    client: reqwest::Client,
+    credentials: Arc<GoogleCredentials>,
+    endpoint: reqwest::Url,
+    project: String,
+    cache: Cache<String, SecretValue>,
+    always_read: bool,
+}
+
+#[derive(Deserialize)]
+struct Response {
+    payload: Option<Payload>,
+}
+
+#[derive(Deserialize)]
+struct Payload {
+    data: Option<String>,
+}
+
+impl GoogleSecretManager {
+    pub fn with_client(
+        client: reqwest::Client,
+        endpoint: reqwest::Url,
+        project: String,
+        environment: Arc<dyn Lookup + Send + Sync>,
+        refresh_interval: Option<Duration>,
+        always_read: bool,
+    ) -> Result<Self, Error> {
+        let credentials = auth::credentials(
+            Some(project.clone()),
+            environment
+                .get(GCS_PATH_SERVICE_ACCOUNT)
+                .map(SecretValue::new),
+            environment,
+        );
+        let ttl = refresh_interval
+            .filter(|ttl| !ttl.is_zero())
+            .unwrap_or(DEFAULT_CACHE_TTL);
+        let cache = Cache::builder()
+            .max_capacity(CACHE_CAPACITY)
+            .time_to_live(ttl)
+            .build();
+        Ok(Self {
+            client,
+            credentials: Arc::new(credentials),
+            endpoint,
+            project,
+            cache,
+            always_read,
+        })
+    }
+
+    pub fn new(
+        environment: Arc<dyn Lookup + Send + Sync>,
+        enterprise_enabled: bool,
+    ) -> Result<Self, Error> {
+        if !enterprise_enabled {
+            return Err(Error::EnterpriseRequired);
+        }
+        let project = environment
+            .get(GOOGLE_SECRET_MANAGER_PROJECT_ID)
+            .ok_or(Error::MissingEnvironment(GOOGLE_SECRET_MANAGER_PROJECT_ID))?;
+        let ttl = environment
+            .get(GOOGLE_SECRET_MANAGER_REFRESH_INTERVAL)
+            .filter(|v| !v.is_empty())
+            .map(|v| v.parse::<i64>().map_err(|_| Error::RefreshInterval))
+            .transpose()?
+            .unwrap_or(
+                environment
+                    .get(SECRET_MANAGER_REFRESH_INTERVAL)
+                    .map(|v| v.parse::<i64>().map_err(|_| Error::RefreshInterval))
+                    .transpose()?
+                    .unwrap_or(DEFAULT_REFRESH_INTERVAL.as_secs() as i64),
+            );
+        let always_read = environment
+            .get(GOOGLE_SECRET_MANAGER_ALWAYS_READ_SECRET_MANAGER)
+            .is_some_and(|v| v.eq_ignore_ascii_case("true"));
+        Self::with_client(
+            reqwest::Client::new(),
+            reqwest::Url::parse("https://secretmanager.googleapis.com").expect("static URL"),
+            project,
+            environment,
+            Some(if ttl < 0 {
+                Duration::from_nanos(1)
+            } else {
+                Duration::from_secs(ttl as u64)
+            }),
+            always_read,
+        )
+    }
+
+    pub async fn get_secret_from_google_secret_manager(
+        &self,
+        name: &str,
+    ) -> Result<Option<Secret>, Error> {
+        if !self.always_read
+            && let Some(cached) = self.cache.get(name).await
+        {
+            return Ok(Some(Secret::String(cached)));
+        }
+        let url = self
+            .endpoint
+            .join(&format!(
+                "/v1/projects/{}/secrets/{}/versions/latest:access",
+                percent_encoding::utf8_percent_encode(
+                    &self.project,
+                    percent_encoding::NON_ALPHANUMERIC
+                ),
+                percent_encoding::utf8_percent_encode(name, percent_encoding::NON_ALPHANUMERIC)
+            ))
+            .map_err(|_| Error::Endpoint)?;
+        let response = self
+            .client
+            .get(url)
+            .headers(self.credentials.request_headers().await?)
+            .send()
+            .await?;
+        if response.status() == reqwest::StatusCode::NOT_FOUND {
+            return Ok(None);
+        }
+        if response.status() != reqwest::StatusCode::OK {
+            return Err(Error::Status(response.status().as_u16()));
+        }
+        let response: Response = response.json().await?;
+        let Some(data) = response.payload.and_then(|payload| payload.data) else {
+            return Err(Error::MissingPayload);
+        };
+        let bytes = STANDARD.decode(data)?;
+        let plaintext = String::from_utf8(bytes).map_err(|_| Error::Utf8)?;
+        let value = SecretValue::new(plaintext);
+        self.cache.insert(name.to_owned(), value.clone()).await;
+        Ok(Some(Secret::String(value)))
+    }
+}

litellm-rust/crates/secrets-google/tests/kms.rs

@@ -0,0 +1,49 @@
+use base64::{Engine, engine::general_purpose::STANDARD};
+use google_cloud_kms_v1::client::KeyManagementService;
+use litellm_secrets_google::GoogleKms;
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{body_json, path},
+};
+
+#[tokio::test]
+async fn google_kms_decrypts_using_the_configured_resource() {
+    let server = MockServer::start().await;
+    let resource = "projects/project/locations/global/keyRings/ring/cryptoKeys/key";
+    Mock::given(path(format!("/v1/{resource}:decrypt")))
+        .and(body_json(
+            serde_json::json!({"ciphertext":STANDARD.encode("encrypted")}),
+        ))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .set_body_json(serde_json::json!({"plaintext":STANDARD.encode(" value\n")})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let client = KeyManagementService::builder()
+        .with_endpoint(server.uri())
+        .with_credentials(google_cloud_auth::credentials::anonymous::Builder::new().build())
+        .with_retry_policy(google_cloud_gax::retry_policy::NeverRetry)
+        .build()
+        .await
+        .unwrap();
+    let manager = GoogleKms::new(client, resource.into());
+    assert_eq!(
+        manager.decrypt(b"encrypted".to_vec()).await.unwrap(),
+        b" value\n"
+    );
+}
+
+#[tokio::test]
+async fn disabled_google_kms_loader_does_not_require_environment_configuration() {
+    use std::sync::Arc;
+    for enabled in [None, Some(false)] {
+        assert!(
+            litellm_secrets_google::load_google_kms(enabled, Arc::new(|_: &str| None))
+                .await
+                .unwrap()
+                .is_none()
+        );
+    }
+}

litellm-rust/crates/secrets-google/tests/secret_manager.rs

@@ -0,0 +1,188 @@
+use std::{sync::Arc, time::Duration};
+
+use base64::{Engine, engine::general_purpose::STANDARD};
+use litellm_secrets_google::{Error, GoogleSecretManager};
+
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{header, path},
+};
+
+fn manager(server: &MockServer, always_read: bool, ttl: Duration) -> GoogleSecretManager {
+    GoogleSecretManager::with_client(
+        reqwest::Client::new(),
+        server.uri().parse().unwrap(),
+        "project".into(),
+        Arc::new(|name: &str| (name == "VERTEX_AI_API_KEY").then(|| "token".into())),
+        Some(ttl),
+        always_read,
+    )
+    .unwrap()
+}
+
+#[rstest::rstest]
+#[case::nonempty("private-value")]
+#[case::empty("")]
+#[tokio::test]
+async fn successful_reads_use_auth_latest_version_and_cache_including_empty_values(
+    #[case] value: &str,
+) {
+    let server = MockServer::start().await;
+    Mock::given(path(
+        "/v1/projects/project/secrets/key/versions/latest:access",
+    ))
+    .and(header("authorization", "Bearer token"))
+    .respond_with(
+        ResponseTemplate::new(200)
+            .set_body_json(serde_json::json!({"payload":{"data":STANDARD.encode(value)}})),
+    )
+    .expect(1)
+    .mount(&server)
+    .await;
+    let manager = manager(&server, false, Duration::from_secs(60));
+    for _ in 0..2 {
+        assert_eq!(
+            manager
+                .get_secret_from_google_secret_manager("key")
+                .await
+                .unwrap()
+                .unwrap()
+                .as_str()
+                .unwrap(),
+            value
+        );
+    }
+}
+
+#[rstest::rstest]
+#[case::not_found(404, serde_json::json!({}))]
+#[case::unauthorized(401, serde_json::json!({}))]
+#[case::forbidden(403, serde_json::json!({}))]
+#[case::throttled(429, serde_json::json!({}))]
+#[case::unavailable(503, serde_json::json!({}))]
+#[case::missing_payload(200, serde_json::json!({"payload":{}}))]
+#[case::invalid_base64(200, serde_json::json!({"payload":{"data":"%%%"}}))]
+#[tokio::test]
+async fn failed_or_missing_reads_are_not_cached(
+    #[case] status: u16,
+    #[case] body: serde_json::Value,
+) {
+    let server = MockServer::start().await;
+    let manager = manager(&server, false, Duration::from_secs(60));
+    let failing = Mock::given(path(
+        "/v1/projects/project/secrets/key/versions/latest:access",
+    ))
+    .respond_with(ResponseTemplate::new(status).set_body_json(body))
+    .expect(1)
+    .mount_as_scoped(&server)
+    .await;
+    let result = manager.get_secret_from_google_secret_manager("key").await;
+    match status {
+        404 => assert_eq!(result.unwrap(), None),
+        200 => assert!(matches!(
+            result,
+            Err(Error::MissingPayload | Error::Base64(_))
+        )),
+        status => assert!(matches!(result, Err(Error::Status(actual)) if actual == status)),
+    }
+    drop(failing);
+    Mock::given(path(
+        "/v1/projects/project/secrets/key/versions/latest:access",
+    ))
+    .respond_with(
+        ResponseTemplate::new(200)
+            .set_body_json(serde_json::json!({"payload":{"data":STANDARD.encode("recovered")}})),
+    )
+    .expect(1)
+    .mount(&server)
+    .await;
+    for _ in 0..2 {
+        assert_eq!(
+            manager
+                .get_secret_from_google_secret_manager("key")
+                .await
+                .unwrap()
+                .unwrap()
+                .as_str(),
+            Some("recovered")
+        );
+    }
+}
+
+#[rstest::rstest]
+#[case::always_read(true, Duration::from_secs(60))]
+#[case::expired_cache(false, Duration::from_millis(1))]
+#[tokio::test]
+async fn always_read_and_expired_cache_fetch_again(
+    #[case] always_read: bool,
+    #[case] ttl: Duration,
+) {
+    let server = MockServer::start().await;
+    Mock::given(path(
+        "/v1/projects/project/secrets/key/versions/latest:access",
+    ))
+    .respond_with(
+        ResponseTemplate::new(200)
+            .set_body_json(serde_json::json!({"payload":{"data":STANDARD.encode("value")}})),
+    )
+    .expect(2)
+    .mount(&server)
+    .await;
+    let manager = manager(&server, always_read, ttl);
+    for _ in 0..2 {
+        tokio::time::sleep(Duration::from_millis(5)).await;
+        assert!(
+            manager
+                .get_secret_from_google_secret_manager("key")
+                .await
+                .unwrap()
+                .is_some()
+        );
+    }
+}
+
+#[test]
+fn google_manager_requires_host_license_and_project_configuration() {
+    assert!(matches!(
+        GoogleSecretManager::new(Arc::new(|_: &str| None), false),
+        Err(Error::EnterpriseRequired)
+    ));
+    assert!(matches!(
+        GoogleSecretManager::new(Arc::new(|_: &str| None), true),
+        Err(Error::MissingEnvironment(
+            "GOOGLE_SECRET_MANAGER_PROJECT_ID"
+        ))
+    ));
+}
+
+#[rstest::rstest]
+#[case("true")]
+#[case("null")]
+#[case("\"text\"")]
+#[case("{\"key\":1}")]
+#[tokio::test]
+async fn cache_preserves_raw_values(#[case] raw: &str) {
+    let server = MockServer::start().await;
+    Mock::given(path(
+        "/v1/projects/project/secrets/key/versions/latest:access",
+    ))
+    .respond_with(
+        ResponseTemplate::new(200)
+            .set_body_json(serde_json::json!({"payload":{"data":STANDARD.encode(raw)}})),
+    )
+    .expect(1)
+    .mount(&server)
+    .await;
+    let manager = manager(&server, false, Duration::from_secs(60));
+    for _ in 0..2 {
+        assert_eq!(
+            manager
+                .get_secret_from_google_secret_manager("key")
+                .await
+                .unwrap()
+                .unwrap()
+                .as_str(),
+            Some(raw)
+        );
+    }
+}

litellm-rust/crates/secrets-types/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "litellm-secrets-types"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+litellm-auth-types.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+veil.workspace = true
+
+[dev-dependencies]
+rstest.workspace = true
+tokio.workspace = true

litellm-rust/crates/secrets-types/src/base_secret_manager.rs

@@ -0,0 +1,58 @@
+use crate::{Error, SecretValue};
+
+pub fn validate_secret_name(name: &str) -> Result<(), Error> {
+    if name.split('/').any(|segment| segment == "..")
+        || name
+            .chars()
+            .any(|c| c.is_control() || matches!(c, '\u{2028}' | '\u{2029}'))
+    {
+        return Err(Error::UnsafeSecretName);
+    }
+    Ok(())
+}
+
+#[expect(
+    async_fn_in_trait,
+    reason = "closed backend dispatch does not require Send bounds on generic rotation"
+)]
+pub trait BaseSecretManager {
+    type Error: From<Error>;
+    type WriteResponse;
+    type DeleteResponse;
+
+    async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Self::Error>;
+    async fn async_write_secret(
+        &self,
+        name: &str,
+        value: &SecretValue,
+        description: Option<&str>,
+    ) -> Result<Self::WriteResponse, Self::Error>;
+    async fn async_delete_secret(
+        &self,
+        name: &str,
+        recovery_window_in_days: i64,
+    ) -> Result<Self::DeleteResponse, Self::Error>;
+}
+
+pub async fn async_rotate_secret<M: BaseSecretManager>(
+    manager: &M,
+    current_name: &str,
+    new_name: &str,
+    value: &SecretValue,
+) -> Result<M::WriteResponse, M::Error> {
+    if manager.async_read_secret(current_name).await?.is_none() {
+        return Err(Error::CurrentSecretMissing.into());
+    }
+    let response = manager
+        .async_write_secret(
+            new_name,
+            value,
+            Some(&format!("Rotated from {current_name}")),
+        )
+        .await?;
+    if manager.async_read_secret(new_name).await?.is_none() {
+        return Err(Error::NewSecretMissing.into());
+    }
+    manager.async_delete_secret(current_name, 7).await?;
+    Ok(response)
+}

litellm-rust/crates/secrets-types/src/config.rs

@@ -0,0 +1,92 @@
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+
+use crate::SecretValue;
+
+#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum KeyManagementSystem {
+    GoogleKms,
+    AzureKeyVault,
+    AwsSecretManager,
+    GoogleSecretManager,
+    HashicorpVault,
+    Cyberark,
+    Local,
+    AwsKms,
+    Custom,
+}
+
+#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum AccessMode {
+    #[default]
+    ReadOnly,
+    WriteOnly,
+    ReadAndWrite,
+}
+
+impl AccessMode {
+    pub fn readable(self) -> bool {
+        matches!(self, Self::ReadOnly | Self::ReadAndWrite)
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[serde(default)]
+pub struct KeyManagementSettings {
+    pub hosted_keys: Option<Vec<String>>,
+    pub store_virtual_keys: Option<bool>,
+    pub prefix_for_stored_virtual_keys: String,
+    pub access_mode: AccessMode,
+    pub primary_secret_name: Option<String>,
+    pub description: Option<String>,
+    pub tags: Option<BTreeMap<String, String>>,
+    pub kms_key_id: Option<String>,
+    pub custom_secret_manager: Option<String>,
+    pub aws_region_name: Option<String>,
+    pub aws_role_name: Option<String>,
+    pub aws_session_name: Option<String>,
+    #[serde(serialize_with = "serialize_secret")]
+    pub aws_external_id: Option<SecretValue>,
+    pub aws_profile_name: Option<String>,
+    #[serde(serialize_with = "serialize_secret")]
+    pub aws_web_identity_token: Option<SecretValue>,
+    pub aws_sts_endpoint: Option<String>,
+    pub replica_regions: Option<Vec<String>>,
+}
+
+impl Default for KeyManagementSettings {
+    fn default() -> Self {
+        Self {
+            hosted_keys: None,
+            store_virtual_keys: Some(false),
+            prefix_for_stored_virtual_keys: "litellm/".into(),
+            access_mode: AccessMode::ReadOnly,
+            primary_secret_name: None,
+            description: None,
+            tags: None,
+            kms_key_id: None,
+            custom_secret_manager: None,
+            aws_region_name: None,
+            aws_role_name: None,
+            aws_session_name: None,
+            aws_external_id: None,
+            aws_profile_name: None,
+            aws_web_identity_token: None,
+            aws_sts_endpoint: None,
+            replica_regions: None,
+        }
+    }
+}
+
+fn serialize_secret<S: serde::Serializer>(
+    value: &Option<SecretValue>,
+    serializer: S,
+) -> Result<S::Ok, S::Error> {
+    value
+        .as_ref()
+        .map(SecretValue::expose)
+        .serialize(serializer)
+}

litellm-rust/crates/secrets-types/src/error.rs

@@ -0,0 +1,9 @@
+#[derive(Debug, thiserror::Error, PartialEq, Eq)]
+pub enum Error {
+    #[error("secret name contains an unsafe path segment or control character")]
+    UnsafeSecretName,
+    #[error("current secret was not found")]
+    CurrentSecretMissing,
+    #[error("new secret could not be verified")]
+    NewSecretMissing,
+}

litellm-rust/crates/secrets-types/src/lib.rs

@@ -0,0 +1,12 @@
+#![forbid(unsafe_code)]
+
+mod base_secret_manager;
+mod config;
+mod error;
+mod value;
+
+pub use base_secret_manager::{BaseSecretManager, async_rotate_secret, validate_secret_name};
+pub use config::{AccessMode, KeyManagementSettings, KeyManagementSystem};
+pub use error::Error;
+pub use litellm_auth_types::SecretValue;
+pub use value::Secret;

litellm-rust/crates/secrets-types/src/value.rs

@@ -0,0 +1,31 @@
+use crate::SecretValue;
+
+#[derive(Clone, PartialEq, Eq, veil::Redact)]
+pub enum Secret {
+    String(SecretValue),
+    Bool(#[redact] bool),
+    Json(#[redact] serde_json::Value),
+}
+
+impl From<SecretValue> for Secret {
+    fn from(value: SecretValue) -> Self {
+        Self::String(value)
+    }
+}
+
+impl Secret {
+    pub fn from_json(value: serde_json::Value) -> Self {
+        match value {
+            serde_json::Value::String(value) => Self::String(SecretValue::new(value)),
+            serde_json::Value::Bool(value) => Self::Bool(value),
+            value => Self::Json(value),
+        }
+    }
+
+    pub fn as_str(&self) -> Option<&str> {
+        match self {
+            Self::String(value) => Some(value.expose()),
+            Self::Bool(_) | Self::Json(_) => None,
+        }
+    }
+}

litellm-rust/crates/secrets-types/tests/config.rs

@@ -0,0 +1,60 @@
+use litellm_secrets_types::{
+    AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
+};
+use serde_json::json;
+
+#[test]
+fn config_preserves_defaults_nulls_and_serialized_names() {
+    let empty: KeyManagementSettings = serde_json::from_value(json!({})).unwrap();
+    assert_eq!(empty, KeyManagementSettings::default());
+    assert_eq!(empty.access_mode, AccessMode::ReadOnly);
+    assert_eq!(empty.store_virtual_keys, Some(false));
+    assert_eq!(empty.prefix_for_stored_virtual_keys, "litellm/");
+    let configured: KeyManagementSettings = serde_json::from_value(json!({
+        "hosted_keys": [], "store_virtual_keys": null, "access_mode": "write_only",
+        "aws_web_identity_token": "private-token", "aws_external_id": "private-id",
+        "tags": {"stage": "test"}, "replica_regions": ["test-region"]
+    }))
+    .unwrap();
+    assert!(!configured.access_mode.readable());
+    assert_eq!(configured.store_virtual_keys, None);
+    assert_eq!(configured.hosted_keys.as_deref(), Some([].as_slice()));
+    assert!(!format!("{configured:?}").contains("private-"));
+    let serialized = serde_json::to_value(&configured).unwrap();
+    assert_eq!(serialized["access_mode"], "write_only");
+    assert_eq!(serialized["aws_web_identity_token"], "private-token");
+    assert_eq!(
+        serde_json::from_value::<KeyManagementSettings>(serialized).unwrap(),
+        configured
+    );
+}
+
+#[rstest::rstest]
+#[case::aws_kms("aws_kms", KeyManagementSystem::AwsKms)]
+#[case::aws_secret_manager("aws_secret_manager", KeyManagementSystem::AwsSecretManager)]
+#[case::google_kms("google_kms", KeyManagementSystem::GoogleKms)]
+#[case::google_secret_manager("google_secret_manager", KeyManagementSystem::GoogleSecretManager)]
+#[case::azure_key_vault("azure_key_vault", KeyManagementSystem::AzureKeyVault)]
+#[case::hashicorp_vault("hashicorp_vault", KeyManagementSystem::HashicorpVault)]
+#[case::cyberark("cyberark", KeyManagementSystem::Cyberark)]
+#[case::custom("custom", KeyManagementSystem::Custom)]
+#[case::local("local", KeyManagementSystem::Local)]
+fn key_management_system_serialization_round_trips(
+    #[case] name: &str,
+    #[case] system: KeyManagementSystem,
+) {
+    assert_eq!(
+        serde_json::from_value::<KeyManagementSystem>(json!(name)).unwrap(),
+        system
+    );
+    assert_eq!(serde_json::to_value(system).unwrap(), name);
+}
+
+#[test]
+fn secret_debug_never_exposes_values() {
+    assert!(
+        !format!("{:?}", Secret::String(SecretValue::new("sensitive-value")))
+            .contains("sensitive-value")
+    );
+    assert!(!format!("{:?}", Secret::Bool(true)).contains("true"));
+}

litellm-rust/crates/secrets-types/tests/rotation.rs

@@ -0,0 +1,105 @@
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use litellm_secrets_types::{
+    BaseSecretManager, Error, SecretValue, async_rotate_secret, validate_secret_name,
+};
+
+struct Manager {
+    step: AtomicUsize,
+    absent_at: Option<usize>,
+}
+
+impl BaseSecretManager for Manager {
+    type Error = Error;
+    type WriteResponse = &'static str;
+    type DeleteResponse = ();
+
+    async fn async_read_secret(&self, name: &str) -> Result<Option<SecretValue>, Error> {
+        let step = self.step.fetch_add(1, Ordering::SeqCst);
+        assert_eq!(name, if step == 0 { "old" } else { "new" });
+        Ok((self.absent_at != Some(step)).then(|| SecretValue::new("value")))
+    }
+
+    async fn async_write_secret(
+        &self,
+        name: &str,
+        value: &SecretValue,
+        description: Option<&str>,
+    ) -> Result<Self::WriteResponse, Error> {
+        assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 1);
+        assert_eq!(name, "new");
+        assert_eq!(value.expose(), "replacement");
+        assert_eq!(description, Some("Rotated from old"));
+        Ok("provider-response")
+    }
+
+    async fn async_delete_secret(
+        &self,
+        name: &str,
+        recovery_window_in_days: i64,
+    ) -> Result<(), Error> {
+        assert_eq!(self.step.fetch_add(1, Ordering::SeqCst), 3);
+        assert_eq!(name, "old");
+        assert_eq!(recovery_window_in_days, 7);
+        Ok(())
+    }
+}
+
+#[tokio::test]
+async fn rotation_verifies_before_deleting_and_returns_provider_response() {
+    let manager = Manager {
+        step: AtomicUsize::new(0),
+        absent_at: None,
+    };
+    assert_eq!(
+        async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement"))
+            .await
+            .unwrap(),
+        "provider-response"
+    );
+    assert_eq!(manager.step.load(Ordering::SeqCst), 4);
+}
+
+#[rstest::rstest]
+#[case::current_secret_missing(0, Error::CurrentSecretMissing, 1)]
+#[case::new_secret_missing(2, Error::NewSecretMissing, 3)]
+#[tokio::test]
+async fn missing_old_or_new_value_stops_rotation_before_deletion(
+    #[case] absent_at: usize,
+    #[case] expected: Error,
+    #[case] calls: usize,
+) {
+    let manager = Manager {
+        step: AtomicUsize::new(0),
+        absent_at: Some(absent_at),
+    };
+    assert_eq!(
+        async_rotate_secret(&manager, "old", "new", &SecretValue::new("replacement"))
+            .await
+            .unwrap_err(),
+        expected
+    );
+    assert_eq!(manager.step.load(Ordering::SeqCst), calls);
+}
+
+#[rstest::rstest]
+#[case::parent("..")]
+#[case::parent_prefix("../x")]
+#[case::parent_segment("x/../y")]
+#[case::parent_suffix("x/..")]
+#[case::line_feed("line\n")]
+#[case::next_line("\u{85}")]
+#[case::line_separator("\u{2028}")]
+#[case::paragraph_separator("\u{2029}")]
+fn names_reject_path_traversal_and_control_characters(#[case] name: &str) {
+    assert_eq!(validate_secret_name(name), Err(Error::UnsafeSecretName));
+}
+
+#[rstest::rstest]
+#[case::embedded_double_dot("release-1.0..2")]
+#[case::path_separator("folder/key")]
+#[case::empty("")]
+#[case::three_dots("...")]
+fn names_allow_safe_values(#[case] name: &str) {
+    assert_eq!(validate_secret_name(name), Ok(()));
+}

litellm-rust/crates/secrets/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "litellm-secrets"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[features]
+default = []
+aws = ["dep:litellm-secrets-aws"]
+google = ["dep:litellm-secrets-google"]
+
+[dependencies]
+litellm-secrets-types.workspace = true
+litellm-secrets-aws = { workspace = true, optional = true }
+litellm-secrets-google = { workspace = true, optional = true }
+litellm-core-utils.workspace = true
+base64.workspace = true
+serde.workspace = true
+strum.workspace = true
+jsonwebtoken.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+reqwest.workspace = true
+moka.workspace = true
+tokio = { workspace = true, features = ["fs"] }
+
+[dev-dependencies]
+rstest.workspace = true
+wiremock = "0.6.5"
+tempfile = "3"
+aws-sdk-kms = "1.120.0"
+google-cloud-kms-v1 = "1.14.0"
+google-cloud-auth.workspace = true

litellm-rust/crates/secrets/README.md

@@ -0,0 +1,11 @@
+# Secret resolution
+
+Construct `SecretManagerState::new(backend, settings)` for a configured manager or use `SecretManagerState::default()` for environment lookups. The configured backend determines its provider identity. Write-only settings and names excluded by `hosted_keys` use the environment directly. `secret_manager_would_be_consulted` follows the same routing decision as resolution
+
+`get_secret` returns `Ok(Some(value))` for a found value, `Ok(None)` when no source contains the value, and `Err(error)` when lookup fails. For managed names, resolution checks the manager, then the environment, then the caller's default. An empty string, `false`, or an explicitly stored JSON null is a found value
+
+Backend failures propagate by default. To allow fallback during a backend failure, construct the resolver with `.with_failure_policy(FailurePolicy::EnvironmentFallback)`. It then tries the environment and default, in that order. If neither exists, the original error is returned. This policy applies to manager lookups. Explicit OIDC references retain their own authentication errors and never fall back to environment secrets under the reference name
+
+`get_secret` preserves value types. `get_secret_str` accepts a string default and rejects boolean or JSON values with `Error::TypeMismatch`. `get_secret_bool` accepts a boolean default and converts strings containing `true` or `false`, ignoring surrounding whitespace and ASCII case. Other strings and JSON values produce `Error::TypeMismatch`. Conversion failures never activate fallback or replace a found value with the default
+
+Provider payloads remain strings unless explicitly selecting a field from an AWS primary JSON secret. Google caches only successfully decoded string payloads, so reads have identical values and types before and after caching. Confirmed absence and failed reads are not cached. AWS resource-not-found responses and Google HTTP 404 responses indicate absence. Other provider errors remain errors, and successful responses without the required payload are malformed responses rather than missing secrets

litellm-rust/crates/secrets/src/error.rs

@@ -0,0 +1,33 @@
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    #[error("encrypted environment value is missing")]
+    MissingCiphertext,
+    #[error("ciphertext is not valid base64 for the configured manager")]
+    InvalidCiphertext,
+    #[error("decrypted value is not UTF-8")]
+    Utf8,
+    #[error("unsupported OIDC provider or missing build feature")]
+    UnsupportedOidc,
+    #[error("OIDC reference requires a provider and audience")]
+    InvalidOidc,
+    #[error("OIDC environment variable is missing")]
+    MissingEnvironment,
+    #[error("OIDC request failed")]
+    OidcHttp,
+    #[error("OIDC provider returned HTTP {0}")]
+    OidcStatus(u16),
+    #[error("OIDC response is invalid")]
+    OidcResponse,
+    #[error("OIDC file path must be absolute and within the credential allowlist")]
+    UnsafeOidcPath,
+    #[error("OIDC file could not be read")]
+    OidcFile,
+    #[error("secret cannot be converted to {expected}")]
+    TypeMismatch { expected: &'static str },
+    #[cfg(feature = "aws")]
+    #[error(transparent)]
+    Aws(#[from] litellm_secrets_aws::Error),
+    #[cfg(feature = "google")]
+    #[error(transparent)]
+    Google(#[from] litellm_secrets_google::Error),
+}

litellm-rust/crates/secrets/src/handler.rs

@@ -0,0 +1,117 @@
+use litellm_core_utils::settings::Lookup;
+
+use crate::{Error, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue};
+
+#[derive(Clone)]
+pub enum SecretManager {
+    Local,
+    #[cfg(feature = "aws")]
+    AwsKms(crate::aws::AwsKms),
+    #[cfg(feature = "aws")]
+    AwsSecretsManagerV2(crate::aws::AwsSecretsManagerV2),
+    #[cfg(feature = "google")]
+    GoogleKms(crate::google::GoogleKms),
+    #[cfg(feature = "google")]
+    GoogleSecretManager(crate::google::GoogleSecretManager),
+}
+
+impl SecretManager {
+    pub fn system(&self) -> KeyManagementSystem {
+        match self {
+            Self::Local => KeyManagementSystem::Local,
+            #[cfg(feature = "aws")]
+            Self::AwsKms(_) => KeyManagementSystem::AwsKms,
+            #[cfg(feature = "aws")]
+            Self::AwsSecretsManagerV2(_) => KeyManagementSystem::AwsSecretManager,
+            #[cfg(feature = "google")]
+            Self::GoogleKms(_) => KeyManagementSystem::GoogleKms,
+            #[cfg(feature = "google")]
+            Self::GoogleSecretManager(_) => KeyManagementSystem::GoogleSecretManager,
+        }
+    }
+}
+
+pub async fn get_secret_from_manager(
+    client: &SecretManager,
+    secret_name: &str,
+    _settings: &KeyManagementSettings,
+    environment: &(dyn Lookup + Send + Sync),
+) -> Result<Option<Secret>, Error> {
+    match client {
+        SecretManager::Local => Ok(environment
+            .get(secret_name)
+            .map(SecretValue::new)
+            .map(Secret::String)),
+        #[cfg(feature = "aws")]
+        SecretManager::AwsKms(client) => {
+            let ciphertext = environment
+                .get(secret_name)
+                .ok_or(Error::MissingCiphertext)?;
+            let plaintext = client
+                .decrypt(decode_ciphertext(&ciphertext, Base64Mode::Permissive)?)
+                .await?;
+            let value = String::from_utf8(plaintext).map_err(|_| Error::Utf8)?;
+            Ok(Some(Secret::String(SecretValue::new(value.trim()))))
+        }
+        #[cfg(feature = "google")]
+        SecretManager::GoogleKms(client) => {
+            let ciphertext = environment
+                .get(secret_name)
+                .ok_or(Error::MissingCiphertext)?;
+            let plaintext = client
+                .decrypt(decode_ciphertext(&ciphertext, Base64Mode::Canonical)?)
+                .await?;
+            let value = String::from_utf8(plaintext).map_err(|_| Error::Utf8)?;
+            Ok(Some(Secret::String(SecretValue::new(value))))
+        }
+        #[cfg(feature = "aws")]
+        SecretManager::AwsSecretsManagerV2(client) => client
+            .read_secret_for_resolver(
+                secret_name,
+                _settings.primary_secret_name.as_deref(),
+                environment,
+            )
+            .await
+            .map_err(Error::from),
+        #[cfg(feature = "google")]
+        SecretManager::GoogleSecretManager(client) => client
+            .get_secret_from_google_secret_manager(secret_name)
+            .await
+            .map_err(Error::from),
+    }
+}
+
+#[cfg(any(feature = "aws", feature = "google"))]
+#[derive(Clone, Copy)]
+enum Base64Mode {
+    #[cfg(feature = "google")]
+    Canonical,
+    #[cfg(feature = "aws")]
+    Permissive,
+}
+
+#[cfg(any(feature = "aws", feature = "google"))]
+fn decode_ciphertext(value: &str, mode: Base64Mode) -> Result<Vec<u8>, Error> {
+    use base64::{Engine, engine::general_purpose::STANDARD};
+    let canonical = match mode {
+        #[cfg(feature = "google")]
+        Base64Mode::Canonical => true,
+        #[cfg(feature = "aws")]
+        Base64Mode::Permissive => false,
+    };
+    let encoded = if canonical {
+        value.to_owned()
+    } else {
+        value
+            .chars()
+            .filter(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '/' | '='))
+            .collect()
+    };
+    let ciphertext = STANDARD
+        .decode(&encoded)
+        .map_err(|_| Error::InvalidCiphertext)?;
+    if canonical && STANDARD.encode(&ciphertext) != encoded {
+        return Err(Error::InvalidCiphertext);
+    }
+    Ok(ciphertext)
+}

litellm-rust/crates/secrets/src/lib.rs

@@ -0,0 +1,21 @@
+#![forbid(unsafe_code)]
+
+mod error;
+mod handler;
+mod oidc;
+mod resolver;
+mod state;
+
+pub use error::Error;
+pub use handler::{SecretManager, get_secret_from_manager};
+pub use litellm_secrets_types::{
+    AccessMode, KeyManagementSettings, KeyManagementSystem, Secret, SecretValue,
+};
+pub use oidc::{OidcProvider, OidcReference, OidcResolver};
+pub use resolver::{FailurePolicy, SecretResolver};
+pub use state::{SecretManagerState, secret_manager_would_be_consulted};
+
+#[cfg(feature = "aws")]
+pub use litellm_secrets_aws as aws;
+#[cfg(feature = "google")]
+pub use litellm_secrets_google as google;

litellm-rust/crates/secrets/src/oidc.rs

@@ -0,0 +1,269 @@
+use std::{
+    path::Path,
+    time::{Duration, SystemTime, UNIX_EPOCH},
+};
+
+use jsonwebtoken::dangerous::insecure_decode_claims;
+use litellm_core_utils::settings::Lookup;
+use moka::future::Cache;
+use serde::Deserialize;
+
+use crate::{Error, SecretValue};
+
+const GOOGLE_TOKEN_MAX_TTL: Duration = Duration::from_secs(3540);
+const GITHUB_TOKEN_TTL: Duration = Duration::from_secs(295);
+const TOKEN_EXPIRY_MARGIN_SECONDS: f64 = 60.0;
+const CIRCLE_OIDC_TOKEN: &str = "CIRCLE_OIDC_TOKEN";
+const CIRCLE_OIDC_TOKEN_V2: &str = "CIRCLE_OIDC_TOKEN_V2";
+const AZURE_FEDERATED_TOKEN_FILE: &str = "AZURE_FEDERATED_TOKEN_FILE";
+const ACTIONS_ID_TOKEN_REQUEST_URL: &str = "ACTIONS_ID_TOKEN_REQUEST_URL";
+const ACTIONS_ID_TOKEN_REQUEST_TOKEN: &str = "ACTIONS_ID_TOKEN_REQUEST_TOKEN";
+const OIDC_ALLOWED_CREDENTIAL_DIRS: &str = "LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS";
+const DEFAULT_CREDENTIAL_DIRS: &str = "/var/run/secrets,/run/secrets";
+
+#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, strum::EnumString, strum::AsRefStr)]
+#[strum(serialize_all = "snake_case")]
+pub enum OidcProvider {
+    Google,
+    #[strum(serialize = "circleci")]
+    CircleCi,
+    #[strum(serialize = "circleci_v2")]
+    CircleCiV2,
+    Github,
+    Azure,
+    File,
+    Env,
+    EnvPath,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub struct OidcReference<'a> {
+    pub provider: OidcProvider,
+    pub audience: &'a str,
+}
+
+impl<'a> TryFrom<&'a str> for OidcReference<'a> {
+    type Error = Error;
+
+    fn try_from(reference: &'a str) -> Result<Self, Error> {
+        let (provider, audience) = reference
+            .strip_prefix("oidc/")
+            .and_then(|body| body.split_once('/'))
+            .ok_or(Error::InvalidOidc)?;
+        Ok(Self {
+            provider: provider.parse().map_err(|_| Error::UnsupportedOidc)?,
+            audience,
+        })
+    }
+}
+
+#[derive(Deserialize)]
+struct OidcTokenClaims {
+    exp: Option<NumericDate>,
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum NumericDate {
+    Number(f64),
+    String(String),
+}
+
+impl NumericDate {
+    fn seconds(self) -> Option<f64> {
+        match self {
+            Self::Number(value) => Some(value),
+            Self::String(value) => value.parse().ok(),
+        }
+        .filter(|value| value.is_finite())
+    }
+}
+
+pub struct OidcResolver {
+    client: reqwest::Client,
+    google_identity_endpoint: reqwest::Url,
+    cache: Cache<String, (SecretValue, SystemTime)>,
+    clock: fn() -> SystemTime,
+}
+
+impl Default for OidcResolver {
+    fn default() -> Self {
+        let client = reqwest::Client::builder()
+            .timeout(Duration::from_secs(600))
+            .connect_timeout(Duration::from_secs(5))
+            .build()
+            .expect("HTTP client configuration");
+        Self::new(
+            client,
+            reqwest::Url::parse("http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity").expect("static URL"),
+        )
+    }
+}
+
+impl OidcResolver {
+    pub fn new(client: reqwest::Client, google_identity_endpoint: reqwest::Url) -> Self {
+        Self {
+            client,
+            google_identity_endpoint,
+            cache: Cache::builder()
+                .max_capacity(200)
+                .time_to_live(GOOGLE_TOKEN_MAX_TTL)
+                .build(),
+            clock: SystemTime::now,
+        }
+    }
+
+    pub fn with_clock(self, clock: fn() -> SystemTime) -> Self {
+        Self { clock, ..self }
+    }
+
+    pub async fn resolve(
+        &self,
+        reference: &str,
+        environment: &(dyn Lookup + Send + Sync),
+    ) -> Result<Option<SecretValue>, Error> {
+        let OidcReference { provider, audience } = reference.try_into()?;
+        match provider {
+            OidcProvider::CircleCi => required_env(environment, CIRCLE_OIDC_TOKEN)
+                .map(SecretValue::new)
+                .map(Some),
+            OidcProvider::CircleCiV2 => required_env(environment, CIRCLE_OIDC_TOKEN_V2)
+                .map(SecretValue::new)
+                .map(Some),
+            OidcProvider::Env => required_env(environment, audience)
+                .map(SecretValue::new)
+                .map(Some),
+            OidcProvider::EnvPath => read_file(&required_env(environment, audience)?)
+                .await
+                .map(Some),
+            OidcProvider::File => read_allowed_file(audience, environment).await.map(Some),
+            OidcProvider::Azure => {
+                if let Some(path) = environment.get(AZURE_FEDERATED_TOKEN_FILE) {
+                    return read_file(&path).await.map(Some);
+                }
+                Err(Error::UnsupportedOidc)
+            }
+            OidcProvider::Github => {
+                let url = required_env(environment, ACTIONS_ID_TOKEN_REQUEST_URL)?;
+                let authorization = required_env(environment, ACTIONS_ID_TOKEN_REQUEST_TOKEN)?;
+                if let Some(value) = self.cached(reference).await {
+                    return Ok(Some(value));
+                }
+                let response = self
+                    .client
+                    .get(url)
+                    .query(&[("audience", audience)])
+                    .bearer_auth(authorization)
+                    .header("Accept", "application/json; api-version=2.0")
+                    .send()
+                    .await
+                    .map_err(|_| Error::OidcHttp)?;
+                if response.status() != reqwest::StatusCode::OK {
+                    return Err(Error::OidcStatus(response.status().as_u16()));
+                }
+                #[derive(Deserialize)]
+                struct Token {
+                    value: Option<SecretValue>,
+                }
+                let token: Token = response.json().await.map_err(|_| Error::OidcResponse)?;
+                if let Some(value) = &token.value {
+                    self.cache
+                        .insert(
+                            reference.to_owned(),
+                            (value.clone(), (self.clock)() + GITHUB_TOKEN_TTL),
+                        )
+                        .await;
+                }
+                Ok(token.value)
+            }
+            OidcProvider::Google => {
+                if !cfg!(feature = "google") {
+                    return Err(Error::UnsupportedOidc);
+                }
+                if let Some(value) = self.cached(reference).await {
+                    return Ok(Some(value));
+                }
+                let response = self
+                    .client
+                    .get(self.google_identity_endpoint.clone())
+                    .query(&[("audience", audience)])
+                    .header("Metadata-Flavor", "Google")
+                    .send()
+                    .await
+                    .map_err(|_| Error::OidcHttp)?;
+                if response.status() != reqwest::StatusCode::OK {
+                    return Err(Error::OidcStatus(response.status().as_u16()));
+                }
+                let token = response.text().await.map_err(|_| Error::OidcResponse)?;
+                let now = (self.clock)();
+                let ttl = oidc_token_cache_ttl(&token, now, GOOGLE_TOKEN_MAX_TTL);
+                let value = SecretValue::new(token);
+                if let Some(ttl) = ttl.filter(|ttl| !ttl.is_zero()) {
+                    self.cache
+                        .insert(reference.to_owned(), (value.clone(), now + ttl))
+                        .await;
+                }
+                Ok(Some(value))
+            }
+        }
+    }
+
+    async fn cached(&self, reference: &str) -> Option<SecretValue> {
+        self.cache
+            .get(reference)
+            .await
+            .and_then(|(value, expires)| ((self.clock)() < expires).then_some(value))
+    }
+}
+
+fn required_env(environment: &dyn Lookup, name: &str) -> Result<String, Error> {
+    environment.get(name).ok_or(Error::MissingEnvironment)
+}
+
+async fn read_file(path: &str) -> Result<SecretValue, Error> {
+    tokio::fs::read_to_string(path)
+        .await
+        .map(|value| SecretValue::new(value.replace("\r\n", "\n").replace('\r', "\n")))
+        .map_err(|_| Error::OidcFile)
+}
+
+async fn read_allowed_file(
+    path: &str,
+    environment: &(dyn Lookup + Sync),
+) -> Result<SecretValue, Error> {
+    if !Path::new(path).is_absolute() {
+        return Err(Error::UnsafeOidcPath);
+    }
+    let resolved = tokio::fs::canonicalize(path)
+        .await
+        .map_err(|_| Error::OidcFile)?;
+    let allowed = environment
+        .get(OIDC_ALLOWED_CREDENTIAL_DIRS)
+        .filter(|v| !v.is_empty())
+        .unwrap_or_else(|| DEFAULT_CREDENTIAL_DIRS.into());
+    for directory in allowed.split(',').map(str::trim).filter(|d| !d.is_empty()) {
+        if let Ok(directory) = tokio::fs::canonicalize(directory).await
+            && resolved.starts_with(directory)
+        {
+            return tokio::fs::read_to_string(&resolved)
+                .await
+                .map(|value| SecretValue::new(value.replace("\r\n", "\n").replace('\r', "\n")))
+                .map_err(|_| Error::OidcFile);
+        }
+    }
+    Err(Error::UnsafeOidcPath)
+}
+
+fn oidc_token_cache_ttl(token: &str, now: SystemTime, max_ttl: Duration) -> Option<Duration> {
+    let fallback = Some(max_ttl);
+    let Ok(claims) = insecure_decode_claims::<OidcTokenClaims>(token) else {
+        return fallback;
+    };
+    let Some(exp) = claims.exp.and_then(NumericDate::seconds) else {
+        return fallback;
+    };
+    let seconds = exp.trunc()
+        - now.duration_since(UNIX_EPOCH).ok()?.as_secs() as f64
+        - TOKEN_EXPIRY_MARGIN_SECONDS;
+    (seconds > 0.0).then(|| Duration::from_secs_f64(seconds.min(max_ttl.as_secs_f64())))
+}

litellm-rust/crates/secrets/src/resolver.rs

@@ -0,0 +1,135 @@
+use std::sync::Arc;
+
+use litellm_core_utils::settings::{Lookup, ProcessEnvironment};
+
+use crate::state::{LookupTarget, normalize_secret_name};
+use crate::{Error, OidcResolver, Secret, SecretManagerState, SecretValue};
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+pub enum FailurePolicy {
+    #[default]
+    Propagate,
+    EnvironmentFallback,
+}
+
+pub struct SecretResolver {
+    state: Arc<SecretManagerState>,
+    environment: Arc<dyn Lookup + Send + Sync>,
+    oidc: OidcResolver,
+    failure_policy: FailurePolicy,
+}
+
+impl Default for SecretResolver {
+    fn default() -> Self {
+        Self::new(
+            Arc::new(SecretManagerState::default()),
+            Arc::new(ProcessEnvironment),
+            OidcResolver::default(),
+        )
+    }
+}
+
+impl SecretResolver {
+    pub fn new(
+        state: Arc<SecretManagerState>,
+        environment: Arc<dyn Lookup + Send + Sync>,
+        oidc: OidcResolver,
+    ) -> Self {
+        Self {
+            state,
+            environment,
+            oidc,
+            failure_policy: FailurePolicy::default(),
+        }
+    }
+
+    pub fn with_failure_policy(self, failure_policy: FailurePolicy) -> Self {
+        Self {
+            failure_policy,
+            ..self
+        }
+    }
+
+    pub async fn get_secret(
+        &self,
+        name: &str,
+        default_value: Option<Secret>,
+    ) -> Result<Option<Secret>, Error> {
+        let name = normalize_secret_name(name);
+        if name.starts_with("oidc/") {
+            return self
+                .oidc
+                .resolve(name, self.environment.as_ref())
+                .await
+                .map(|value| value.map(Secret::String).or(default_value));
+        }
+        let LookupTarget::Manager { backend, settings } = self.state.lookup_target(name) else {
+            return Ok(self.environment_secret(name).or(default_value));
+        };
+        match crate::get_secret_from_manager(backend, name, settings, self.environment.as_ref())
+            .await
+        {
+            Ok(value) => Ok(value
+                .or_else(|| self.environment_secret(name))
+                .or(default_value)),
+            Err(error) => match self.failure_policy {
+                FailurePolicy::Propagate => Err(error),
+                FailurePolicy::EnvironmentFallback => self
+                    .environment_secret(name)
+                    .or(default_value)
+                    .map(Some)
+                    .ok_or(error),
+            },
+        }
+    }
+
+    fn environment_secret(&self, name: &str) -> Option<Secret> {
+        self.environment
+            .get(name)
+            .map(SecretValue::new)
+            .map(Secret::String)
+    }
+
+    pub async fn get_secret_str(
+        &self,
+        name: &str,
+        default_value: Option<SecretValue>,
+    ) -> Result<Option<SecretValue>, Error> {
+        match self
+            .get_secret(name, default_value.map(Secret::String))
+            .await?
+        {
+            Some(Secret::String(value)) => Ok(Some(value)),
+            None => Ok(None),
+            Some(Secret::Bool(_) | Secret::Json(_)) => {
+                Err(Error::TypeMismatch { expected: "string" })
+            }
+        }
+    }
+
+    pub async fn get_secret_bool(
+        &self,
+        name: &str,
+        default_value: Option<bool>,
+    ) -> Result<Option<bool>, Error> {
+        match self
+            .get_secret(name, default_value.map(Secret::Bool))
+            .await?
+        {
+            Some(Secret::Bool(value)) => Ok(Some(value)),
+            Some(Secret::String(value)) => {
+                match value.expose().trim().to_ascii_lowercase().as_str() {
+                    "true" => Ok(Some(true)),
+                    "false" => Ok(Some(false)),
+                    _ => Err(Error::TypeMismatch {
+                        expected: "boolean",
+                    }),
+                }
+            }
+            Some(Secret::Json(_)) => Err(Error::TypeMismatch {
+                expected: "boolean",
+            }),
+            None => Ok(None),
+        }
+    }
+}

litellm-rust/crates/secrets/src/state.rs

@@ -0,0 +1,59 @@
+use crate::{KeyManagementSettings, KeyManagementSystem, SecretManager};
+
+pub(crate) enum LookupTarget<'a> {
+    Environment,
+    Manager {
+        backend: &'a SecretManager,
+        settings: &'a KeyManagementSettings,
+    },
+}
+
+pub(crate) fn normalize_secret_name(name: &str) -> &str {
+    name.strip_prefix("os.environ/").unwrap_or(name)
+}
+
+#[derive(Clone, Default)]
+pub struct SecretManagerState {
+    manager: Option<(SecretManager, KeyManagementSettings)>,
+}
+
+impl SecretManagerState {
+    pub fn new(backend: SecretManager, settings: KeyManagementSettings) -> Self {
+        Self {
+            manager: Some((backend, settings)),
+        }
+    }
+
+    pub fn system(&self) -> Option<KeyManagementSystem> {
+        self.backend().map(SecretManager::system)
+    }
+
+    pub fn settings(&self) -> Option<&KeyManagementSettings> {
+        self.manager.as_ref().map(|(_, settings)| settings)
+    }
+
+    pub fn backend(&self) -> Option<&SecretManager> {
+        self.manager.as_ref().map(|(backend, _)| backend)
+    }
+
+    pub(crate) fn lookup_target(&self, name: &str) -> LookupTarget<'_> {
+        match &self.manager {
+            Some((backend, settings))
+                if backend.system() != KeyManagementSystem::Local
+                    && settings.access_mode.readable()
+                    && settings
+                        .hosted_keys
+                        .as_ref()
+                        .is_none_or(|keys| keys.iter().any(|key| key == name)) =>
+            {
+                LookupTarget::Manager { backend, settings }
+            }
+            _ => LookupTarget::Environment,
+        }
+    }
+}
+
+pub fn secret_manager_would_be_consulted(state: &SecretManagerState, name: &str) -> bool {
+    let name = normalize_secret_name(name);
+    !name.starts_with("oidc/") && matches!(state.lookup_target(name), LookupTarget::Manager { .. })
+}

litellm-rust/crates/secrets/tests/handler.rs

@@ -0,0 +1,107 @@
+#[cfg(feature = "aws")]
+#[tokio::test]
+async fn aws_handler_reads_ciphertext_decodes_trims_and_redacts() {
+    use aws_sdk_kms::{
+        Client,
+        config::{BehaviorVersion, Credentials, Region},
+    };
+    use base64::{Engine, engine::general_purpose::STANDARD};
+    use litellm_secrets::{
+        Error, KeyManagementSettings, SecretManager, aws::AwsKms, get_secret_from_manager,
+    };
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::body_json};
+
+    let server = MockServer::start().await;
+    Mock::given(body_json(
+        serde_json::json!({"CiphertextBlob": STANDARD.encode("encrypted")}),
+    ))
+    .respond_with(
+        ResponseTemplate::new(200)
+            .set_body_json(serde_json::json!({"Plaintext":STANDARD.encode(" value\n")})),
+    )
+    .expect(1)
+    .mount(&server)
+    .await;
+    let client = Client::from_conf(
+        aws_sdk_kms::Config::builder()
+            .behavior_version(BehaviorVersion::latest())
+            .region(Region::new("us-east-1"))
+            .credentials_provider(Credentials::new("test", "test", None, None, "test"))
+            .endpoint_url(server.uri())
+            .build(),
+    );
+    let manager = SecretManager::AwsKms(AwsKms::new(client));
+    let settings = KeyManagementSettings::default();
+    let value = get_secret_from_manager(&manager, "KEY", &settings, &|name: &str| {
+        assert_eq!(name, "KEY");
+        Some(format!(" {}\n", STANDARD.encode("encrypted")))
+    })
+    .await
+    .unwrap()
+    .unwrap();
+    assert_eq!(value.as_str(), Some("value"));
+    assert!(!format!("{value:?}").contains("value"));
+    assert!(matches!(
+        get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| None).await,
+        Err(Error::MissingCiphertext)
+    ));
+    assert!(matches!(
+        get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| Some("abc".into())).await,
+        Err(Error::InvalidCiphertext)
+    ));
+}
+
+#[cfg(feature = "google")]
+#[tokio::test]
+async fn google_handler_requires_canonical_base64_and_preserves_plaintext_whitespace() {
+    use base64::{Engine, engine::general_purpose::STANDARD};
+    use google_cloud_kms_v1::client::KeyManagementService;
+    use litellm_secrets::{
+        Error, KeyManagementSettings, SecretManager, get_secret_from_manager, google::GoogleKms,
+    };
+    use wiremock::{
+        Mock, MockServer, ResponseTemplate,
+        matchers::{body_json, path},
+    };
+
+    let server = MockServer::start().await;
+    let resource = "projects/project/locations/global/keyRings/ring/cryptoKeys/key";
+    Mock::given(path(format!("/v1/{resource}:decrypt")))
+        .and(body_json(
+            serde_json::json!({"ciphertext":STANDARD.encode("encrypted")}),
+        ))
+        .respond_with(
+            ResponseTemplate::new(200)
+                .set_body_json(serde_json::json!({"plaintext":STANDARD.encode(" value\n")})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let client = KeyManagementService::builder()
+        .with_endpoint(server.uri())
+        .with_credentials(google_cloud_auth::credentials::anonymous::Builder::new().build())
+        .build()
+        .await
+        .unwrap();
+    let manager = SecretManager::GoogleKms(GoogleKms::new(client, resource.into()));
+    let settings = KeyManagementSettings::default();
+    let value = get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| {
+        Some(STANDARD.encode("encrypted"))
+    })
+    .await
+    .unwrap()
+    .unwrap();
+    assert_eq!(value.as_str(), Some(" value\n"));
+    assert!(matches!(
+        get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| Some(format!(
+            " {}",
+            STANDARD.encode("encrypted")
+        )))
+        .await,
+        Err(Error::InvalidCiphertext)
+    ));
+    assert!(matches!(
+        get_secret_from_manager(&manager, "KEY", &settings, &|_: &str| None).await,
+        Err(Error::MissingCiphertext)
+    ));
+}

litellm-rust/crates/secrets/tests/oidc.rs

@@ -0,0 +1,295 @@
+use std::{collections::BTreeMap, sync::Arc};
+
+use litellm_core_utils::settings::Lookup;
+use litellm_secrets::{Error, OidcResolver, Secret, SecretManagerState, SecretResolver};
+use wiremock::{
+    Mock, MockServer, ResponseTemplate,
+    matchers::{header, method, path, query_param},
+};
+
+fn environment(pairs: &[(&str, &str)]) -> Arc<dyn Lookup + Send + Sync> {
+    let values: BTreeMap<String, String> = pairs
+        .iter()
+        .map(|(k, v)| (k.to_string(), v.to_string()))
+        .collect();
+    Arc::new(move |name: &str| values.get(name).cloned())
+}
+
+#[rstest::rstest]
+#[case::environment("oidc/env/TOKEN", "true")]
+#[case::circleci("oidc/circleci/audience", "circle")]
+#[case::circleci_v2("oidc/circleci_v2/audience", "circle-v2")]
+#[tokio::test]
+async fn environment_sources_resolve_expected_value(
+    #[case] reference: &str,
+    #[case] expected: &str,
+) {
+    let env = environment(&[
+        ("TOKEN", "true"),
+        ("CIRCLE_OIDC_TOKEN", "circle"),
+        ("CIRCLE_OIDC_TOKEN_V2", "circle-v2"),
+    ]);
+    assert_eq!(
+        OidcResolver::default()
+            .resolve(reference, env.as_ref())
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        expected
+    );
+}
+
+#[tokio::test]
+async fn environment_sources_bypass_boolean_conversion_and_defaults() {
+    let env = environment(&[("TOKEN", "true")]);
+    let oidc = OidcResolver::default();
+    let resolver = SecretResolver::new(Arc::new(SecretManagerState::default()), env, oidc);
+    assert_eq!(
+        resolver
+            .get_secret_str("os.environ/oidc/env/TOKEN", None)
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "true"
+    );
+    assert_eq!(
+        resolver
+            .get_secret_bool("oidc/env/TOKEN", None)
+            .await
+            .unwrap(),
+        Some(true)
+    );
+    assert!(matches!(
+        resolver
+            .get_secret("oidc/env/MISSING", Some(Secret::Bool(true)))
+            .await,
+        Err(Error::MissingEnvironment)
+    ));
+    assert!(matches!(
+        resolver.get_secret("oidc/invalid", None).await,
+        Err(Error::InvalidOidc)
+    ));
+}
+
+#[tokio::test]
+async fn github_requests_are_authenticated_cached_and_revalidate_environment() {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .and(path("/token"))
+        .and(query_param("audience", "https://service/oidc/path"))
+        .and(header("authorization", "Bearer request-token"))
+        .and(header("accept", "application/json; api-version=2.0"))
+        .respond_with(
+            ResponseTemplate::new(200).set_body_json(serde_json::json!({"value":"identity-token"})),
+        )
+        .expect(1)
+        .mount(&server)
+        .await;
+    let env = environment(&[
+        (
+            "ACTIONS_ID_TOKEN_REQUEST_URL",
+            &format!("{}/token", server.uri()),
+        ),
+        ("ACTIONS_ID_TOKEN_REQUEST_TOKEN", "request-token"),
+    ]);
+    let oidc = OidcResolver::default();
+    for _ in 0..2 {
+        assert_eq!(
+            oidc.resolve("oidc/github/https://service/oidc/path", env.as_ref())
+                .await
+                .unwrap()
+                .unwrap()
+                .expose(),
+            "identity-token"
+        );
+    }
+    assert!(matches!(
+        oidc.resolve(
+            "oidc/github/https://service/oidc/path",
+            environment(&[]).as_ref()
+        )
+        .await,
+        Err(Error::MissingEnvironment)
+    ));
+}
+
+#[tokio::test]
+async fn file_allowlist_resolves_symlinks_while_environment_paths_remain_explicit() {
+    let allowed = tempfile::tempdir().unwrap();
+    let outside = tempfile::tempdir().unwrap();
+    let token = allowed.path().join("token");
+    let private = outside.path().join("private");
+    std::fs::write(&token, "token\r\n").unwrap();
+    std::fs::write(&private, "outside").unwrap();
+    let env = environment(&[
+        (
+            "LITELLM_OIDC_ALLOWED_CREDENTIAL_DIRS",
+            allowed.path().to_str().unwrap(),
+        ),
+        ("PATH_TOKEN", private.to_str().unwrap()),
+        ("AZURE_FEDERATED_TOKEN_FILE", token.to_str().unwrap()),
+    ]);
+    let oidc = OidcResolver::default();
+    assert_eq!(
+        oidc.resolve(&format!("oidc/file/{}", token.display()), env.as_ref())
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "token\n"
+    );
+    assert!(matches!(
+        oidc.resolve("oidc/file/relative", env.as_ref()).await,
+        Err(Error::UnsafeOidcPath)
+    ));
+    assert!(matches!(
+        oidc.resolve(&format!("oidc/file/{}", private.display()), env.as_ref())
+            .await,
+        Err(Error::UnsafeOidcPath)
+    ));
+    assert_eq!(
+        oidc.resolve("oidc/env_path/PATH_TOKEN", env.as_ref())
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "outside"
+    );
+    assert_eq!(
+        oidc.resolve("oidc/azure/scope", env.as_ref())
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "token\n"
+    );
+    #[cfg(unix)]
+    {
+        let link = allowed.path().join("link");
+        std::os::unix::fs::symlink(&private, &link).unwrap();
+        assert!(matches!(
+            oidc.resolve(&format!("oidc/file/{}", link.display()), env.as_ref())
+                .await,
+            Err(Error::UnsafeOidcPath)
+        ));
+    }
+}
+
+#[cfg(feature = "google")]
+#[rstest::rstest]
+#[case::at_refresh_boundary(serde_json::json!(1060), 2)]
+#[case::beyond_refresh_boundary(serde_json::json!(1061), 1)]
+#[case::already_expired(serde_json::json!(999), 2)]
+#[case::string_expiry(serde_json::json!("999"), 2)]
+#[case::fractional_expiry(serde_json::json!(1060.9), 2)]
+#[case::negative_expiry(serde_json::json!(-1), 2)]
+#[case::null_expiry(serde_json::Value::Null, 1)]
+#[case::unreadable_expiry(serde_json::json!("invalid"), 1)]
+#[case::nonfinite_expiry(serde_json::json!("NaN"), 1)]
+#[tokio::test]
+async fn google_expiry_caps_cache_and_preserves_audience(
+    #[case] expiry: serde_json::Value,
+    #[case] calls: u64,
+) {
+    use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};
+    use std::time::{Duration, SystemTime, UNIX_EPOCH};
+    fn now() -> SystemTime {
+        UNIX_EPOCH + Duration::from_secs(1000)
+    }
+    let server = MockServer::start().await;
+    let token = format!(
+        "{}.{}.signature",
+        URL_SAFE_NO_PAD.encode(serde_json::json!({"alg":"RS256","typ":"JWT"}).to_string()),
+        URL_SAFE_NO_PAD.encode(serde_json::json!({"exp":expiry}).to_string())
+    );
+    Mock::given(method("GET"))
+        .and(header("metadata-flavor", "Google"))
+        .and(query_param("audience", "https://service/oidc/path"))
+        .respond_with(ResponseTemplate::new(200).set_body_string(&token))
+        .expect(calls)
+        .mount(&server)
+        .await;
+    let oidc =
+        OidcResolver::new(reqwest::Client::new(), server.uri().parse().unwrap()).with_clock(now);
+    for _ in 0..2 {
+        assert_eq!(
+            oidc.resolve(
+                "oidc/google/https://service/oidc/path",
+                environment(&[]).as_ref()
+            )
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+            token
+        );
+    }
+}
+
+#[cfg(not(feature = "google"))]
+#[tokio::test]
+async fn google_oidc_requires_its_build_feature() {
+    assert!(matches!(
+        OidcResolver::default()
+            .resolve("oidc/google/audience", environment(&[]).as_ref())
+            .await,
+        Err(Error::UnsupportedOidc)
+    ));
+}
+
+#[tokio::test]
+async fn azure_oidc_without_a_token_file_requires_an_unimplemented_backend() {
+    assert!(matches!(
+        OidcResolver::default()
+            .resolve("oidc/azure/scope", environment(&[]).as_ref())
+            .await,
+        Err(Error::UnsupportedOidc)
+    ));
+}
+
+#[rstest::rstest]
+#[case::missing_prefix("env/TOKEN", false)]
+#[case::missing_audience_separator("oidc/env", false)]
+#[case::unknown_provider("oidc/unknown/TOKEN", true)]
+#[tokio::test]
+async fn invalid_references_fail_before_environment_lookup(
+    #[case] reference: &str,
+    #[case] unsupported: bool,
+) {
+    let error = OidcResolver::default()
+        .resolve(reference, &|_: &str| {
+            panic!("invalid reference reached environment lookup")
+        })
+        .await
+        .unwrap_err();
+    assert!(matches!(error, Error::UnsupportedOidc) == unsupported);
+    assert!(matches!(error, Error::InvalidOidc) != unsupported);
+}
+
+#[cfg(feature = "google")]
+#[rstest::rstest]
+#[case::opaque("opaque-token")]
+#[case::missing_expiry("header.e30.signature")]
+#[tokio::test]
+async fn unreadable_expiry_keeps_python_cache_fallback(#[case] token: &str) {
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .respond_with(ResponseTemplate::new(200).set_body_string(token))
+        .expect(1)
+        .mount(&server)
+        .await;
+    let resolver = OidcResolver::new(reqwest::Client::new(), server.uri().parse().unwrap());
+    for _ in 0..2 {
+        assert_eq!(
+            resolver
+                .resolve("oidc/google/audience", environment(&[]).as_ref())
+                .await
+                .unwrap()
+                .unwrap()
+                .expose(),
+            token,
+        );
+    }
+}

litellm-rust/crates/secrets/tests/resolution.rs

@@ -0,0 +1,368 @@
+use std::sync::Arc;
+
+use litellm_secrets::{
+    Error, KeyManagementSettings, OidcResolver, Secret, SecretManager, SecretManagerState,
+    SecretResolver, SecretValue, secret_manager_would_be_consulted,
+};
+
+fn resolver(value: Option<&str>, configured: bool) -> SecretResolver {
+    let state = if configured {
+        SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default())
+    } else {
+        SecretManagerState::default()
+    };
+    let value = value.map(str::to_owned);
+    SecretResolver::new(
+        Arc::new(state),
+        Arc::new(move |_: &str| value.clone()),
+        OidcResolver::default(),
+    )
+}
+
+#[rstest::rstest]
+#[case("true", Some(true))]
+#[case(" FALSE ", Some(false))]
+#[case("(True)", None)]
+#[case("False # comment", None)]
+#[case("1", None)]
+#[case("secret", None)]
+#[tokio::test]
+async fn conversion_is_explicit_and_independent_of_manager_configuration(
+    #[case] input: &str,
+    #[case] boolean: Option<bool>,
+    #[values(false, true)] configured: bool,
+) {
+    let resolver = resolver(Some(input), configured);
+    assert_eq!(
+        resolver.get_secret("key", None).await.unwrap(),
+        Some(Secret::String(SecretValue::new(input)))
+    );
+    assert_eq!(
+        resolver
+            .get_secret_str("key", None)
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        input
+    );
+    match boolean {
+        Some(value) => assert_eq!(
+            resolver.get_secret_bool("key", None).await.unwrap(),
+            Some(value)
+        ),
+        None => assert!(matches!(
+            resolver.get_secret_bool("key", Some(true)).await,
+            Err(Error::TypeMismatch {
+                expected: "boolean"
+            })
+        )),
+    }
+}
+
+#[rstest::rstest]
+#[tokio::test]
+async fn defaults_apply_only_to_absence(#[values(false, true)] configured: bool) {
+    let missing = resolver(None, configured);
+    assert_eq!(missing.get_secret("key", None).await.unwrap(), None);
+    assert_eq!(
+        missing.get_secret_bool("key", Some(false)).await.unwrap(),
+        Some(false)
+    );
+    assert_eq!(
+        missing
+            .get_secret_str("key", Some(SecretValue::new("default")))
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "default"
+    );
+    for value in [
+        Secret::Bool(false),
+        Secret::from_json(serde_json::json!({"key":1})),
+        Secret::from_json(serde_json::Value::Null),
+    ] {
+        assert_eq!(
+            missing
+                .get_secret("key", Some(value.clone()))
+                .await
+                .unwrap(),
+            Some(value)
+        );
+    }
+    assert_eq!(
+        resolver(Some(""), configured)
+            .get_secret_str("key", Some(SecretValue::new("default")))
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        ""
+    );
+}
+
+#[tokio::test]
+async fn prefix_is_removed_once_and_local_manager_is_not_consulted() {
+    let state = SecretManagerState::new(SecretManager::Local, KeyManagementSettings::default());
+    assert_eq!(
+        state.system(),
+        Some(litellm_secrets::KeyManagementSystem::Local)
+    );
+    assert!(!secret_manager_would_be_consulted(
+        &state,
+        "os.environ/os.environ/KEY"
+    ));
+    let resolver = SecretResolver::new(
+        Arc::new(state),
+        Arc::new(|name: &str| (name == "os.environ/KEY").then(|| "value".into())),
+        OidcResolver::default(),
+    );
+    assert_eq!(
+        resolver
+            .get_secret_str("os.environ/os.environ/KEY", None)
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "value"
+    );
+}
+
+#[tokio::test]
+async fn resolver_future_can_run_on_a_tokio_worker() {
+    let resolver = resolver(Some("worker-value"), false);
+    let result = tokio::spawn(async move { resolver.get_secret_str("KEY", None).await })
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(result.unwrap().expose(), "worker-value");
+}
+
+#[cfg(feature = "aws")]
+mod aws {
+    use super::*;
+    use litellm_secrets::{AccessMode, FailurePolicy, aws::AwsSecretsManagerV2};
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
+
+    fn state(server: &MockServer, settings: KeyManagementSettings) -> SecretManagerState {
+        let endpoint = server.uri();
+        let environment = Arc::new(move |name: &str| match name {
+            "AWS_REGION_NAME" => Some("us-east-1".into()),
+            "AWS_ACCESS_KEY_ID" | "AWS_SECRET_ACCESS_KEY" => Some("test".into()),
+            "AWS_BEDROCK_RUNTIME_ENDPOINT" => Some(endpoint.clone()),
+            _ => None,
+        });
+        let manager =
+            AwsSecretsManagerV2::load_aws_secret_manager(Some(true), settings.clone(), environment)
+                .unwrap()
+                .unwrap();
+        SecretManagerState::new(SecretManager::AwsSecretsManagerV2(manager), settings)
+    }
+
+    #[rstest::rstest]
+    #[case::missing(400, serde_json::json!({"__type":"ResourceNotFoundException"}), false)]
+    #[case::denied(400, serde_json::json!({"__type":"AccessDeniedException"}), true)]
+    #[case::malformed(200, serde_json::json!({}), true)]
+    #[tokio::test]
+    async fn failure_policy_preserves_errors_and_fallback_precedence(
+        #[case] status: u16,
+        #[case] body: serde_json::Value,
+        #[case] fails: bool,
+        #[values(FailurePolicy::Propagate, FailurePolicy::EnvironmentFallback)]
+        policy: FailurePolicy,
+        #[values(None, Some("environment"))] environment: Option<&'static str>,
+        #[values(None, Some("default"))] default: Option<&str>,
+    ) {
+        let server = MockServer::start().await;
+        Mock::given(method("POST"))
+            .respond_with(ResponseTemplate::new(status).set_body_json(body))
+            .expect(1)
+            .mount(&server)
+            .await;
+        let resolver = SecretResolver::new(
+            Arc::new(state(&server, KeyManagementSettings::default())),
+            Arc::new(move |_: &str| environment.map(str::to_owned)),
+            OidcResolver::default(),
+        )
+        .with_failure_policy(policy);
+        let result = resolver
+            .get_secret_str("KEY", default.map(SecretValue::new))
+            .await;
+        let fallback = environment.or(default);
+        if fails && (policy == FailurePolicy::Propagate || fallback.is_none()) {
+            assert!(matches!(result, Err(Error::Aws(_))));
+        } else {
+            assert_eq!(result.unwrap().as_ref().map(SecretValue::expose), fallback);
+        }
+    }
+
+    #[rstest::rstest]
+    #[case::boolean(serde_json::json!(false))]
+    #[case::object(serde_json::json!({"key":1}))]
+    #[case::null(serde_json::Value::Null)]
+    #[case::string(serde_json::json!("true"))]
+    #[tokio::test]
+    async fn typed_values_survive_resolution_and_accessors_reject_wrong_types(
+        #[case] value: serde_json::Value,
+    ) {
+        let server = MockServer::start().await;
+        Mock::given(method("POST"))
+            .respond_with(ResponseTemplate::new(200).set_body_json(
+                serde_json::json!({"SecretString":serde_json::json!({"KEY":value}).to_string()}),
+            ))
+            .expect(3)
+            .mount(&server)
+            .await;
+        let settings = KeyManagementSettings {
+            primary_secret_name: Some("primary".into()),
+            ..Default::default()
+        };
+        let resolver = SecretResolver::new(
+            Arc::new(state(&server, settings)),
+            Arc::new(|_: &str| Some("fallback".into())),
+            OidcResolver::default(),
+        );
+        assert_eq!(
+            resolver
+                .get_secret("KEY", Some(Secret::Bool(true)))
+                .await
+                .unwrap(),
+            Some(Secret::from_json(value.clone()))
+        );
+        match &value {
+            serde_json::Value::String(text) => assert_eq!(
+                resolver
+                    .get_secret_str("KEY", None)
+                    .await
+                    .unwrap()
+                    .unwrap()
+                    .expose(),
+                text
+            ),
+            _ => assert!(matches!(
+                resolver.get_secret_str("KEY", None).await,
+                Err(Error::TypeMismatch { expected: "string" })
+            )),
+        }
+        match value {
+            serde_json::Value::Bool(boolean) => assert_eq!(
+                resolver.get_secret_bool("KEY", None).await.unwrap(),
+                Some(boolean)
+            ),
+            serde_json::Value::String(_) => assert_eq!(
+                resolver.get_secret_bool("KEY", None).await.unwrap(),
+                Some(true)
+            ),
+            _ => assert!(matches!(
+                resolver.get_secret_bool("KEY", None).await,
+                Err(Error::TypeMismatch {
+                    expected: "boolean"
+                })
+            )),
+        }
+    }
+
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn gating_prediction_matches_actual_lookup(
+        #[values(AccessMode::ReadOnly, AccessMode::WriteOnly, AccessMode::ReadAndWrite)]
+        access_mode: AccessMode,
+        #[values(None, Some(vec![]), Some(vec!["KEY".into()]))] hosted_keys: Option<Vec<String>>,
+        #[values("os.environ/KEY", "os.environ/oidc/env/KEY")] name: &str,
+    ) {
+        let server = MockServer::start().await;
+        let expected = name == "os.environ/KEY"
+            && access_mode.readable()
+            && hosted_keys
+                .as_ref()
+                .is_none_or(|keys| keys.iter().any(|key| key == "KEY"));
+        Mock::given(method("POST"))
+            .respond_with(
+                ResponseTemplate::new(200)
+                    .set_body_json(serde_json::json!({"SecretString":"remote"})),
+            )
+            .expect(u64::from(expected))
+            .mount(&server)
+            .await;
+        let state = state(
+            &server,
+            KeyManagementSettings {
+                access_mode,
+                hosted_keys,
+                ..Default::default()
+            },
+        );
+        assert!(state.backend().is_some());
+        assert_eq!(state.settings().unwrap().access_mode, access_mode);
+        assert_eq!(secret_manager_would_be_consulted(&state, name), expected);
+        let resolver = SecretResolver::new(
+            Arc::new(state),
+            Arc::new(|_: &str| Some("environment".into())),
+            OidcResolver::default(),
+        );
+        assert_eq!(
+            resolver
+                .get_secret_str(name, None)
+                .await
+                .unwrap()
+                .unwrap()
+                .expose(),
+            if expected { "remote" } else { "environment" }
+        );
+    }
+}
+
+#[cfg(feature = "google")]
+#[rstest::rstest]
+#[case::missing(404)]
+#[case::failure(503)]
+#[tokio::test]
+async fn google_resolver_distinguishes_absence_from_failure(#[case] status: u16) {
+    use litellm_secrets::{FailurePolicy, google::GoogleSecretManager};
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::method};
+    let server = MockServer::start().await;
+    Mock::given(method("GET"))
+        .respond_with(ResponseTemplate::new(status))
+        .expect(2)
+        .mount(&server)
+        .await;
+    let environment: Arc<dyn litellm_core_utils::settings::Lookup + Send + Sync> =
+        Arc::new(|name: &str| match name {
+            "VERTEX_AI_API_KEY" => Some("token".into()),
+            "KEY" => Some("environment".into()),
+            _ => None,
+        });
+    let manager = GoogleSecretManager::with_client(
+        reqwest::Client::new(),
+        server.uri().parse().unwrap(),
+        "project".into(),
+        environment.clone(),
+        None,
+        false,
+    )
+    .unwrap();
+    let state = SecretManagerState::new(
+        SecretManager::GoogleSecretManager(manager),
+        KeyManagementSettings::default(),
+    );
+    let resolver = SecretResolver::new(Arc::new(state), environment, OidcResolver::default());
+    let result = resolver.get_secret_str("KEY", None).await;
+    if status == 404 {
+        assert_eq!(result.unwrap().unwrap().expose(), "environment");
+    } else {
+        assert!(
+            matches!(result, Err(Error::Google(litellm_secrets::google::Error::Status(actual))) if actual == status)
+        );
+    }
+    assert_eq!(
+        resolver
+            .with_failure_policy(FailurePolicy::EnvironmentFallback)
+            .get_secret_str("KEY", None)
+            .await
+            .unwrap()
+            .unwrap()
+            .expose(),
+        "environment"
+    );
+}

litellm-rust/crates/token-counter-fast/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "litellm-token-counter-fast"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+base64.workspace = true
+rustc-hash = "2.1.3"
+thiserror.workspace = true
+tokenizers.workspace = true
+unicode-normalization-alignments = "0.1.12"
+
+[dev-dependencies]
+rand.workspace = true
+rstest.workspace = true
+serde.workspace = true
+serde_json.workspace = true

litellm-rust/crates/token-counter-fast/src/byte_level.rs

@@ -473,13 +473,13 @@ mod tests {
             _ => unreachable!(),
         }
         assert!(ByteLevelCounter::detect(&anthropic_tokenizer).is_none());
-        let counter = crate::TokenCounter::from_json(
+        let counter = crate::FastTokenizer::from_json(
             &anthropic_tokenizer.to_string(false).expect("serialize"),
         )
         .expect("load");
         for text in ["", "Hello WORLD!  ＡＢ fi Ⅳ", "<EOT> stop"] {
             assert_eq!(
-                counter.count_text(text).expect("count"),
+                counter.count_tokens(text).expect("count"),
                 reference_count(&anthropic_tokenizer, text)
             );
         }
@@ -545,7 +545,7 @@ mod tests {
                 .rstrip(rstrip)])
             .expect("add token");
         let fast = ByteLevelCounter::detect(&anthropic_tokenizer).expect("supported");
-        let counter = crate::TokenCounter::from_json(
+        let counter = crate::FastTokenizer::from_json(
             &anthropic_tokenizer.to_string(false).expect("serialize"),
         )
         .expect("load");
@@ -557,7 +557,7 @@ mod tests {
         ] {
             assert_eq!(fast.count(&anthropic_tokenizer, text), None);
             assert_eq!(
-                counter.count_text(text).expect("count"),
+                counter.count_tokens(text).expect("count"),
                 reference_count(&anthropic_tokenizer, text)
             );
         }
@@ -571,17 +571,17 @@ mod tests {
         assert_eq!(fast.count(&tokenizer, "hello"), None);
         assert!(tokenizer.encode_fast("hello", true).is_err());
         let counter =
-            crate::TokenCounter::from_json(&tokenizer.to_string(false).expect("serialize"))
+            crate::FastTokenizer::from_json(&tokenizer.to_string(false).expect("serialize"))
                 .expect("load");
         assert!(matches!(
-            counter.count_text("hello"),
+            counter.count_tokens("hello"),
             Err(crate::Error::Encode(_))
         ));
     }
 
     #[rstest]
     fn shared_counter_matches_encoder_across_threads(anthropic_tokenizer: Tokenizer) {
-        let counter = crate::TokenCounter::from_json(
+        let counter = crate::FastTokenizer::from_json(
             &anthropic_tokenizer.to_string(false).expect("serialize"),
         )
         .expect("load");
@@ -598,7 +598,7 @@ mod tests {
                 scope.spawn(move || {
                     for _ in 0..100 {
                         for (text, count) in inputs.iter().zip(expected) {
-                            assert_eq!(counter.count_text(text).expect("count"), count);
+                            assert_eq!(counter.count_tokens(text).expect("count"), count);
                         }
                     }
                 });
@@ -614,10 +614,10 @@ mod tests {
         let fast = ByteLevelCounter::detect(&anthropic_tokenizer).expect("supported");
         assert_eq!(reference_count(&anthropic_tokenizer, "ABCD EFGH"), 1);
         assert_eq!(fast.count(&anthropic_tokenizer, "ABCD EFGH"), None);
-        let counter = crate::TokenCounter::from_json(
+        let counter = crate::FastTokenizer::from_json(
             &anthropic_tokenizer.to_string(false).expect("serialize"),
         )
         .expect("load");
-        assert_eq!(counter.count_text("ABCD EFGH").expect("count"), 1);
+        assert_eq!(counter.count_tokens("ABCD EFGH").expect("count"), 1);
     }
 }

litellm-rust/crates/token-counter-fast/src/error.rs

@@ -0,0 +1,13 @@
+use thiserror::Error as ThisError;
+
+#[derive(Debug, ThisError)]
+pub enum Error {
+    #[error("failed to load tokenizer: {0}")]
+    Load(#[source] tokenizers::Error),
+    #[error("failed to load tokenizer: tiktoken rank file: {0}")]
+    Ranks(String),
+    #[error("failed to load tokenizer: Unicode character classes are unavailable")]
+    UnicodeClasses,
+    #[error("tokenization failed: {0}")]
+    Encode(#[source] tokenizers::Error),
+}

litellm-rust/crates/token-counter-fast/src/lib.rs

@@ -0,0 +1,70 @@
+#![forbid(unsafe_code)]
+
+mod byte_level;
+mod cl100k;
+mod error;
+mod o200k;
+mod scanner;
+mod tiktoken;
+mod unicode_classes;
+
+use byte_level::ByteLevelCounter;
+use scanner::{SplitPattern, TiktokenCounter};
+
+pub use error::Error;
+
+enum Encoder {
+    HuggingFace {
+        tokenizer: Box<tokenizers::Tokenizer>,
+        byte_level: Option<ByteLevelCounter>,
+    },
+    Tiktoken(TiktokenCounter),
+}
+
+pub struct FastTokenizer(Encoder);
+
+impl FastTokenizer {
+    pub fn from_json(json: &str) -> Result<Self, Error> {
+        let tokenizer = json.parse::<tokenizers::Tokenizer>().map_err(Error::Load)?;
+        let byte_level = ByteLevelCounter::detect(&tokenizer);
+        Ok(Self(Encoder::HuggingFace {
+            tokenizer: Box::new(tokenizer),
+            byte_level,
+        }))
+    }
+
+    pub fn from_cl100k_ranks(ranks: &str) -> Result<Self, Error> {
+        Self::from_ranks(SplitPattern::Cl100k, ranks)
+    }
+
+    pub fn from_o200k_ranks(ranks: &str) -> Result<Self, Error> {
+        Self::from_ranks(SplitPattern::O200k, ranks)
+    }
+
+    fn from_ranks(split: SplitPattern, ranks: &str) -> Result<Self, Error> {
+        TiktokenCounter::from_ranks(split, ranks)
+            .map(Encoder::Tiktoken)
+            .map(Self)
+    }
+
+    pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
+        match &self.0 {
+            Encoder::Tiktoken(counter) => Ok(counter.count(text)),
+            Encoder::HuggingFace {
+                tokenizer,
+                byte_level,
+            } => {
+                if let Some(count) = byte_level
+                    .as_ref()
+                    .and_then(|counter| counter.count(tokenizer, text))
+                {
+                    return Ok(count);
+                }
+                tokenizer
+                    .encode_fast(text, true)
+                    .map(|encoding| encoding.len())
+                    .map_err(Error::Encode)
+            }
+        }
+    }
+}

litellm-rust/crates/token-counter-fast/src/tiktoken.rs

@@ -0,0 +1,240 @@
+//! tiktoken's byte-level BPE: a rank file of `base64(token) rank` lines and
+//! the merge loop that turns one regex piece into tokens. The merge order is
+//! tiktoken's (lowest rank first, leftmost pair on ties) so the token count is
+//! identical, but pairs are tracked in a heap so a long piece costs
+//! `O(n log n)` instead of tiktoken's `O(n^2)`.
+
+use std::cmp::Reverse;
+use std::collections::BinaryHeap;
+
+use base64::Engine;
+use base64::engine::general_purpose::STANDARD;
+use rustc_hash::FxHashMap;
+
+use crate::Error;
+
+type Rank = u32;
+
+const NO_RANK: Rank = Rank::MAX;
+const END: usize = usize::MAX;
+
+pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
+
+impl MergeRanks {
+    pub(super) fn parse(text: &str) -> Result<Self, Error> {
+        let ranks = text
+            .lines()
+            .filter(|line| !line.is_empty())
+            .map(parse_line)
+            .collect::<Result<FxHashMap<_, _>, _>>()?;
+        if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
+            return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
+        }
+        Ok(Self(ranks))
+    }
+
+    fn rank(&self, bytes: &[u8]) -> Rank {
+        self.0.get(bytes).copied().unwrap_or(NO_RANK)
+    }
+
+    /// Token count of one regex piece, as `encode_ordinary` would produce.
+    pub(super) fn count_piece(&self, piece: &[u8], scratch: &mut MergeScratch) -> usize {
+        if piece.len() < 2 || self.0.contains_key(piece) {
+            return 1;
+        }
+        scratch.reset(piece.len());
+        for start in 0..piece.len() - 1 {
+            scratch.set_rank(start, self.rank(&piece[start..start + 2]));
+        }
+        let mut parts = piece.len();
+        while let Some(Reverse((rank, start))) = scratch.heap.pop() {
+            if scratch.next[start] == END || scratch.rank[start] != rank {
+                continue;
+            }
+            let merged = scratch.next[start];
+            let after = scratch.next[merged];
+            scratch.next[merged] = END;
+            scratch.next[start] = after;
+            parts -= 1;
+            if after < piece.len() {
+                scratch.prev[after] = start;
+                scratch.set_rank(start, self.rank(&piece[start..scratch.end(after)]));
+            } else {
+                scratch.rank[start] = NO_RANK;
+            }
+            let before = scratch.prev[start];
+            if before != END {
+                scratch.set_rank(before, self.rank(&piece[before..scratch.end(start)]));
+            }
+        }
+        parts
+    }
+}
+
+fn parse_line(line: &str) -> Result<(Box<[u8]>, Rank), Error> {
+    let (token, rank) = line
+        .split_once(' ')
+        .ok_or_else(|| Error::Ranks(format!("line without a rank: {line:?}")))?;
+    let bytes = STANDARD
+        .decode(token)
+        .map_err(|error| Error::Ranks(format!("token is not base64: {error}")))?;
+    let rank = rank
+        .parse()
+        .map_err(|error| Error::Ranks(format!("rank is not an integer: {error}")))?;
+    if rank == NO_RANK {
+        return Err(Error::Ranks(format!("rank {rank} is reserved")));
+    }
+    Ok((bytes.into_boxed_slice(), rank))
+}
+
+/// Buffers reused across the pieces of one text. Parts are addressed by the
+/// byte offset they start at, which also gives the leftmost-pair tie break.
+#[derive(Default)]
+pub(super) struct MergeScratch {
+    next: Vec<usize>,
+    prev: Vec<usize>,
+    rank: Vec<Rank>,
+    heap: BinaryHeap<Reverse<(Rank, usize)>>,
+}
+
+impl MergeScratch {
+    fn reset(&mut self, len: usize) {
+        self.next.clear();
+        self.next.extend(1..=len);
+        self.prev.clear();
+        self.prev.push(END);
+        self.prev.extend(0..len - 1);
+        self.rank.clear();
+        self.rank.resize(len, NO_RANK);
+        self.heap.clear();
+    }
+
+    fn end(&self, start: usize) -> usize {
+        self.next[start]
+    }
+
+    fn set_rank(&mut self, start: usize, rank: Rank) {
+        self.rank[start] = rank;
+        if rank != NO_RANK {
+            self.heap.push(Reverse((rank, start)));
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rand::rngs::StdRng;
+    use rand::{Rng, SeedableRng};
+
+    use super::*;
+
+    fn ranks() -> MergeRanks {
+        let path = concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/../../../litellm/litellm_core_utils/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
+        );
+        MergeRanks::parse(&std::fs::read_to_string(path).expect("cl100k rank file is in the repo"))
+            .expect("rank file parses")
+    }
+
+    /// tiktoken's `_byte_pair_merge`, transcribed, as the reference.
+    fn reference_count(ranks: &MergeRanks, piece: &[u8]) -> usize {
+        if piece.len() < 2 || ranks.0.contains_key(piece) {
+            return 1;
+        }
+        let mut parts: Vec<(usize, Rank)> = (0..piece.len() - 1)
+            .map(|index| (index, ranks.rank(&piece[index..index + 2])))
+            .chain([(piece.len() - 1, NO_RANK), (piece.len(), NO_RANK)])
+            .collect();
+        let get_rank = |parts: &[(usize, Rank)], index: usize| {
+            if index + 3 < parts.len() {
+                ranks.rank(&piece[parts[index].0..parts[index + 3].0])
+            } else {
+                NO_RANK
+            }
+        };
+        loop {
+            let Some(index) = parts[..parts.len() - 1]
+                .iter()
+                .enumerate()
+                .filter(|(_, (_, rank))| *rank != NO_RANK)
+                .min_by_key(|(index, (_, rank))| (*rank, *index))
+                .map(|(index, _)| index)
+            else {
+                return parts.len() - 1;
+            };
+            if index > 0 {
+                parts[index - 1].1 = get_rank(&parts, index - 1);
+            }
+            parts[index].1 = get_rank(&parts, index);
+            parts.remove(index + 1);
+        }
+    }
+
+    #[test]
+    fn every_byte_is_a_token() {
+        let ranks = ranks();
+        assert_eq!(ranks.0.len(), 100_256);
+        assert!((0..=u8::MAX).all(|byte| ranks.rank(&[byte]) != NO_RANK));
+    }
+
+    #[test]
+    fn heap_merge_matches_tiktokens_merge_loop() {
+        let ranks = ranks();
+        let mut scratch = MergeScratch::default();
+        let mut rng = StdRng::seed_from_u64(99);
+        let alphabet = b" abcdeorstn.,'\n\xc3\xa9\xe2\x82\xac0123";
+        for _ in 0..20_000 {
+            let piece: Vec<u8> = (0..rng.gen_range(1..24))
+                .map(|_| alphabet[rng.gen_range(0..alphabet.len())])
+                .collect();
+            assert_eq!(
+                ranks.count_piece(&piece, &mut scratch),
+                reference_count(&ranks, &piece),
+                "piece {:?}",
+                String::from_utf8_lossy(&piece)
+            );
+        }
+    }
+
+    #[test]
+    fn long_repeated_runs_cost_close_to_linear() {
+        let ranks = ranks();
+        let mut scratch = MergeScratch::default();
+        let mut time = |len: usize| {
+            let piece = vec![b' '; len];
+            let started = std::time::Instant::now();
+            assert!(ranks.count_piece(&piece, &mut scratch) > 0);
+            started.elapsed()
+        };
+        let small = (0..3).map(|_| time(1 << 14)).min().unwrap();
+        let large = time(1 << 18);
+        assert!(
+            large < small * 64,
+            "{small:?} for 2^14 bytes, {large:?} for 2^18"
+        );
+    }
+
+    #[test]
+    fn reserved_merge_rank_is_rejected() {
+        let bytes = (0..=u8::MAX)
+            .map(|byte| format!("{} {byte}\n", STANDARD.encode([byte])))
+            .collect::<String>();
+        let rank_file = format!("{bytes}{} {NO_RANK}\n", STANDARD.encode(b"ab"));
+        assert!(matches!(
+            MergeRanks::parse(&rank_file),
+            Err(Error::Ranks(_))
+        ));
+        let valid_rank_file = format!("{bytes}{} {}\n", STANDARD.encode(b"ab"), NO_RANK - 1);
+        let ranks = MergeRanks::parse(&valid_rank_file).unwrap();
+        assert_eq!(ranks.count_piece(b"aab", &mut MergeScratch::default()), 2);
+    }
+
+    #[test]
+    fn malformed_rank_files_are_rejected() {
+        assert!(MergeRanks::parse("IQ==").is_err());
+        assert!(MergeRanks::parse("IQ== x").is_err());
+        assert!(MergeRanks::parse("!!! 1").is_err());
+        assert!(MergeRanks::parse("IQ== 1").is_err());
+    }
+}

litellm-rust/crates/token-counter-fast/tests/fixtures/generate.py

@@ -2,8 +2,8 @@
 
 Run from the repository root with the project environment, once per encoding:
 
-    uv run --no-sync python litellm-rust/crates/token-counter/tests/fixtures/generate.py cl100k_base
-    uv run --no-sync python litellm-rust/crates/token-counter/tests/fixtures/generate.py o200k_base
+    uv run --no-sync python litellm-rust/crates/token-counter-fast/tests/fixtures/generate.py cl100k_base
+    uv run --no-sync python litellm-rust/crates/token-counter-fast/tests/fixtures/generate.py o200k_base
 
 `<encoding>/texts.jsonl` holds `{"text", "tokens", "pieces"}` lines: `tokens`
 counted with `tiktoken.get_encoding(name).encode(text, disallowed_special=())`,

litellm-rust/crates/token-counter-huggingface/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "litellm-token-counter-huggingface"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+thiserror.workspace = true
+tokenizers.workspace = true

litellm-rust/crates/token-counter-huggingface/src/error.rs

@@ -0,0 +1,9 @@
+use thiserror::Error as ThisError;
+
+#[derive(Debug, ThisError)]
+pub enum Error {
+    #[error("failed to load tokenizer: {0}")]
+    Load(#[source] tokenizers::Error),
+    #[error("tokenization failed: {0}")]
+    Encode(#[source] tokenizers::Error),
+}

litellm-rust/crates/token-counter-huggingface/src/lib.rs

@@ -0,0 +1,23 @@
+#![forbid(unsafe_code)]
+
+mod error;
+
+pub use error::Error;
+
+pub struct HuggingFaceTokenizer(Box<tokenizers::Tokenizer>);
+
+impl HuggingFaceTokenizer {
+    pub fn from_json(json: &str) -> Result<Self, Error> {
+        json.parse::<tokenizers::Tokenizer>()
+            .map(Box::new)
+            .map(Self)
+            .map_err(Error::Load)
+    }
+
+    pub fn count_tokens(&self, text: &str) -> Result<usize, Error> {
+        self.0
+            .encode_fast(text, true)
+            .map(|encoding| encoding.len())
+            .map_err(Error::Encode)
+    }
+}

litellm-rust/crates/token-counter-tiktoken/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "litellm-token-counter-tiktoken"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+thiserror.workspace = true
+tiktoken-rs.workspace = true

litellm-rust/crates/token-counter-tiktoken/src/error.rs

@@ -0,0 +1,5 @@
+use thiserror::Error as ThisError;
+
+#[derive(Debug, ThisError)]
+#[error("unsupported tokenizer: {0}")]
+pub struct UnsupportedTokenizer(pub String);

litellm-rust/crates/token-counter-tiktoken/src/lib.rs

@@ -0,0 +1,70 @@
+#![forbid(unsafe_code)]
+
+mod error;
+
+pub use error::UnsupportedTokenizer;
+
+pub struct TiktokenTokenizer(&'static tiktoken_rs::CoreBPE);
+
+impl TiktokenTokenizer {
+    pub fn from_name(name: &str) -> Result<Self, UnsupportedTokenizer> {
+        let tokenizer = match name {
+            "cl100k_base" => tiktoken_rs::cl100k_base_singleton(),
+            "o200k_base" => tiktoken_rs::o200k_base_singleton(),
+            "o200k_harmony" => tiktoken_rs::o200k_harmony_singleton(),
+            "p50k_base" => tiktoken_rs::p50k_base_singleton(),
+            "p50k_edit" => tiktoken_rs::p50k_edit_singleton(),
+            "r50k_base" | "gpt2" => tiktoken_rs::r50k_base_singleton(),
+            _ => return Err(UnsupportedTokenizer(name.to_owned())),
+        };
+        Ok(Self(tokenizer))
+    }
+
+    pub fn count_tokens(&self, text: &str) -> usize {
+        self.0.count_ordinary(text)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn named_encodings_match_their_reference_counts() {
+        let encodings = [
+            ("cl100k_base", tiktoken_rs::cl100k_base_singleton()),
+            ("o200k_base", tiktoken_rs::o200k_base_singleton()),
+            ("o200k_harmony", tiktoken_rs::o200k_harmony_singleton()),
+            ("p50k_base", tiktoken_rs::p50k_base_singleton()),
+            ("p50k_edit", tiktoken_rs::p50k_edit_singleton()),
+            ("r50k_base", tiktoken_rs::r50k_base_singleton()),
+            ("gpt2", tiktoken_rs::r50k_base_singleton()),
+        ];
+        let texts = [
+            "",
+            "Hello, how are you today?",
+            "é e\u{301} 漢字 ع ३ 🙂 ＡfiⅣ",
+            "  def function():\n    return 123456789\r\n",
+            "<|endoftext|><|fim_prefix|><|start|>assistant<|message|>",
+        ];
+        for (name, reference) in encodings {
+            let counter = TiktokenTokenizer::from_name(name).unwrap();
+            for text in texts {
+                assert_eq!(
+                    counter.count_tokens(text),
+                    reference.encode_ordinary(text).len(),
+                    "{name}: {text:?}",
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn unsupported_encoding_preserves_its_name() {
+        let Err(UnsupportedTokenizer(name)) = TiktokenTokenizer::from_name("unknown-encoding")
+        else {
+            panic!("unknown encoding must be rejected");
+        };
+        assert_eq!(name, "unknown-encoding");
+    }
+}

litellm-rust/crates/token-counter/Cargo.toml

@@ -5,26 +5,34 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[features]
+default = ["fast", "huggingface", "tiktoken"]
+fast = ["dep:litellm-token-counter-fast"]
+huggingface = ["dep:litellm-token-counter-huggingface"]
+tiktoken = ["dep:litellm-token-counter-tiktoken"]
+
 [dependencies]
-base64.workspace = true
 indexmap = { version = "2.14.0", features = ["serde"] }
 itoa = "1.0"
-rustc-hash = "2.1.3"
+litellm-token-counter-fast = { workspace = true, optional = true }
+litellm-token-counter-huggingface = { workspace = true, optional = true }
+litellm-token-counter-tiktoken = { workspace = true, optional = true }
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true
-tokenizers = { version = "0.23.1", default-features = false, features = ["onig"] }
-unicode-normalization-alignments = "0.1.12"
 
 [dev-dependencies]
 criterion.workspace = true
 rand.workspace = true
 rstest.workspace = true
+tokenizers.workspace = true
 
 [[bench]]
 name = "token_counter"
 harness = false
+required-features = ["fast"]
 
 [[bench]]
 name = "allocations"
 harness = false
+required-features = ["fast"]

litellm-rust/crates/token-counter/README.md

@@ -0,0 +1,23 @@
+# Token counting
+
+`Tokenizer` is the text-counting interface. `TokenCounter` applies LiteLLM request, message, and tool accounting using any implementation of that interface
+
+The `fast` feature provides `fast::FastTokenizer` from `litellm-token-counter-fast`. `TokenCounter::from_json_fast` uses this implementation
+
+The `huggingface` feature provides `huggingface::HuggingFaceTokenizer` through the upstream `tokenizers` library. `TokenCounter::from_json` uses this implementation
+
+The `tiktoken` feature provides `tiktoken::TiktokenTokenizer` through `tiktoken-rs`. Select an encoding with `TokenCounter::from_tiktoken`. The supported names are `cl100k_base`, `o200k_base`, `o200k_harmony`, `p50k_base`, `p50k_edit`, `r50k_base`, and `gpt2`
+
+All three backends are enabled by default. The Python extension builds with `fast` only, which keeps the wheel at the size it had before the split. With `default-features = false`, callers can supply their own `Tokenizer` to `TokenCounter::new` without compiling a built-in backend
+
+Budget checks, cost calculation, and the `max_tokens` adjustment policy belong to `litellm-core-utils`. The counter does not own prices, budgets, or request limits
+
+Run the feature matrix with:
+
+```sh
+cargo test -p litellm-token-counter
+cargo test -p litellm-token-counter --no-default-features
+cargo test -p litellm-token-counter --no-default-features --features fast
+cargo test -p litellm-token-counter --no-default-features --features huggingface
+cargo test -p litellm-token-counter --no-default-features --features tiktoken
+```

litellm-rust/crates/token-counter/benches/allocations.rs

@@ -87,7 +87,7 @@ fn main() {
         },
     );
 
-    let counter = TokenCounter::from_json(TOKENIZER_JSON).expect("tokenizer loads");
+    let counter = TokenCounter::from_json_fast(TOKENIZER_JSON).expect("tokenizer loads");
     let object = CountableRequest::parse(OBJECT_BODY).expect("object request parses");
     counter
         .count_request(&object)

litellm-rust/crates/token-counter/benches/token_counter.rs

@@ -42,7 +42,7 @@ fn inputs(tokenizer: &Tokenizer) -> Vec<(&'static str, String)> {
 }
 
 fn token_counter(c: &mut Criterion) {
-    let counter = TokenCounter::from_json(TOKENIZER_JSON).expect("token counter should load");
+    let counter = TokenCounter::from_json_fast(TOKENIZER_JSON).expect("token counter should load");
     let tokenizer = TOKENIZER_JSON
         .parse::<Tokenizer>()
         .expect("reference tokenizer should load");

litellm-rust/crates/token-counter/src/counter.rs

@@ -1,9 +1,7 @@
 use serde::Serialize;
 
 use crate::Error;
-use crate::byte_level::ByteLevelCounter;
 use crate::python_json;
-use crate::scanner::{SplitPattern, TiktokenCounter};
 use crate::tools::format_function_definitions;
 use crate::types::{
     ContentBlock, ContentItem, CountableRequest, Message, MessageContent, TextValue, ToolChoice,
@@ -24,73 +22,22 @@ pub struct InputTokenCount {
     pub input_tokens: usize,
 }
 
-enum Encoder {
-    HuggingFace {
-        tokenizer: Box<tokenizers::Tokenizer>,
-        byte_level: Option<ByteLevelCounter>,
-    },
-    Tiktoken(TiktokenCounter),
-}
-
 /// A loaded tokenizer plus the message accounting Python applies on top of
 /// it. Encoding is CPU-bound and synchronous; hosts run it off their event
 /// loop.
 pub struct TokenCounter {
-    encoder: Encoder,
+    encoder: Box<dyn crate::Tokenizer>,
 }
 
 impl TokenCounter {
-    /// Load a HuggingFace `tokenizer.json` document. The host reads the file.
-    pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
-        let tokenizer = tokenizer_json
-            .parse::<tokenizers::Tokenizer>()
-            .map_err(Error::Load)?;
-        let byte_level = ByteLevelCounter::detect(&tokenizer);
-        Ok(Self {
-            encoder: Encoder::HuggingFace {
-                tokenizer: Box::new(tokenizer),
-                byte_level,
-            },
-        })
-    }
-
-    /// Load tiktoken's `cl100k_base` rank file (`base64(token) rank` lines).
-    /// The host reads the file.
-    pub fn from_cl100k_ranks(rank_file: &str) -> Result<Self, Error> {
-        Self::from_tiktoken_ranks(SplitPattern::Cl100k, rank_file)
-    }
-
-    /// Load tiktoken's `o200k_base` rank file (`base64(token) rank` lines).
-    /// The host reads the file.
-    pub fn from_o200k_ranks(rank_file: &str) -> Result<Self, Error> {
-        Self::from_tiktoken_ranks(SplitPattern::O200k, rank_file)
-    }
-
-    fn from_tiktoken_ranks(split: SplitPattern, rank_file: &str) -> Result<Self, Error> {
-        Ok(Self {
-            encoder: Encoder::Tiktoken(TiktokenCounter::from_ranks(split, rank_file)?),
-        })
+    pub fn new(tokenizer: impl crate::Tokenizer + 'static) -> Self {
+        Self {
+            encoder: Box::new(tokenizer),
+        }
     }
 
     pub fn count_text(&self, text: &str) -> Result<usize, Error> {
-        match &self.encoder {
-            Encoder::Tiktoken(counter) => Ok(counter.count(text)),
-            Encoder::HuggingFace {
-                tokenizer,
-                byte_level,
-            } => {
-                if let Some(count) = byte_level
-                    .as_ref()
-                    .and_then(|counter| counter.count(tokenizer, text))
-                {
-                    return Ok(count);
-                }
-                tokenizer
-                    .encode_fast(text, true)
-                    .map(|encoding| encoding.len())
-                    .map_err(Error::Encode)
-            }
-        }
+        self.encoder.count_tokens(text)
     }
 
     /// Mirrors the host's key precedence: `messages`, then `prompt`, then

litellm-rust/crates/token-counter/src/error.rs

@@ -4,8 +4,10 @@ use thiserror::Error as ThisError;
 
 #[derive(Debug, ThisError)]
 pub enum Error {
+    #[error("unsupported tokenizer: {0}")]
+    UnsupportedTokenizer(String),
     #[error("failed to load tokenizer: {0}")]
-    Load(#[source] tokenizers::Error),
+    Load(#[source] Box<dyn std::error::Error + Send + Sync>),
     #[error("failed to load tokenizer: tiktoken rank file: {0}")]
     Ranks(String),
     #[error("failed to load tokenizer: Unicode character classes are unavailable")]
@@ -29,7 +31,7 @@ pub enum Error {
     #[error("unsupported by the rust token counter: serialized text value is not UTF-8: {0}")]
     JsonUtf8(#[source] FromUtf8Error),
     #[error("tokenization failed: {0}")]
-    Encode(#[source] tokenizers::Error),
+    Encode(#[source] Box<dyn std::error::Error + Send + Sync>),
     #[error("token counting task failed: {0}")]
     Task(String),
 }

litellm-rust/crates/token-counter/src/fast.rs

@@ -0,0 +1,41 @@
+use litellm_token_counter_fast::Error as BackendError;
+pub use litellm_token_counter_fast::FastTokenizer;
+
+use crate::{Error, TokenCounter, Tokenizer};
+
+impl TokenCounter {
+    pub fn from_json_fast(tokenizer_json: &str) -> Result<Self, Error> {
+        FastTokenizer::from_json(tokenizer_json)
+            .map(Self::new)
+            .map_err(Error::from)
+    }
+
+    pub fn from_cl100k_ranks(rank_file: &str) -> Result<Self, Error> {
+        FastTokenizer::from_cl100k_ranks(rank_file)
+            .map(Self::new)
+            .map_err(Error::from)
+    }
+
+    pub fn from_o200k_ranks(rank_file: &str) -> Result<Self, Error> {
+        FastTokenizer::from_o200k_ranks(rank_file)
+            .map(Self::new)
+            .map_err(Error::from)
+    }
+}
+
+impl Tokenizer for FastTokenizer {
+    fn count_tokens(&self, text: &str) -> Result<usize, Error> {
+        FastTokenizer::count_tokens(self, text).map_err(Error::from)
+    }
+}
+
+impl From<BackendError> for Error {
+    fn from(error: BackendError) -> Self {
+        match error {
+            BackendError::Load(source) => Self::Load(source),
+            BackendError::Ranks(message) => Self::Ranks(message),
+            BackendError::UnicodeClasses => Self::UnicodeClasses,
+            BackendError::Encode(source) => Self::Encode(source),
+        }
+    }
+}

litellm-rust/crates/token-counter/src/huggingface.rs

@@ -0,0 +1,27 @@
+use litellm_token_counter_huggingface::Error as BackendError;
+pub use litellm_token_counter_huggingface::HuggingFaceTokenizer;
+
+use crate::{Error, TokenCounter, Tokenizer};
+
+impl TokenCounter {
+    pub fn from_json(tokenizer_json: &str) -> Result<Self, Error> {
+        HuggingFaceTokenizer::from_json(tokenizer_json)
+            .map(Self::new)
+            .map_err(Error::from)
+    }
+}
+
+impl Tokenizer for HuggingFaceTokenizer {
+    fn count_tokens(&self, text: &str) -> Result<usize, Error> {
+        HuggingFaceTokenizer::count_tokens(self, text).map_err(Error::from)
+    }
+}
+
+impl From<BackendError> for Error {
+    fn from(error: BackendError) -> Self {
+        match error {
+            BackendError::Load(source) => Self::Load(source),
+            BackendError::Encode(source) => Self::Encode(source),
+        }
+    }
+}

litellm-rust/crates/token-counter/src/lib.rs

@@ -4,18 +4,21 @@
 
 #![forbid(unsafe_code)]
 
-mod byte_level;
-mod cl100k;
 mod counter;
 mod error;
-mod o200k;
 mod python_json;
-mod scanner;
-mod tiktoken;
+mod tokenizer;
 mod tools;
 mod types;
-mod unicode_classes;
+
+#[cfg(feature = "fast")]
+pub mod fast;
+#[cfg(feature = "huggingface")]
+pub mod huggingface;
+#[cfg(feature = "tiktoken")]
+pub mod tiktoken;
 
 pub use counter::{InputTokenCount, TokenCounter};
 pub use error::Error;
+pub use tokenizer::Tokenizer;
 pub use types::CountableRequest;

litellm-rust/crates/token-counter/src/tiktoken.rs

@@ -1,222 +1,24 @@
-//! tiktoken's byte-level BPE: a rank file of `base64(token) rank` lines and
-//! the merge loop that turns one regex piece into tokens. The merge order is
-//! tiktoken's (lowest rank first, leftmost pair on ties) so the token count is
-//! identical, but pairs are tracked in a heap so a long piece costs
-//! `O(n log n)` instead of tiktoken's `O(n^2)`.
+pub use litellm_token_counter_tiktoken::TiktokenTokenizer;
+use litellm_token_counter_tiktoken::UnsupportedTokenizer;
 
-use std::cmp::Reverse;
-use std::collections::BinaryHeap;
+use crate::{Error, TokenCounter, Tokenizer};
 
-use base64::Engine;
-use base64::engine::general_purpose::STANDARD;
-use rustc_hash::FxHashMap;
-
-use crate::Error;
-
-type Rank = u32;
-
-const NO_RANK: Rank = Rank::MAX;
-const END: usize = usize::MAX;
-
-pub(super) struct MergeRanks(FxHashMap<Box<[u8]>, Rank>);
-
-impl MergeRanks {
-    pub(super) fn parse(text: &str) -> Result<Self, Error> {
-        let ranks = text
-            .lines()
-            .filter(|line| !line.is_empty())
-            .map(parse_line)
-            .collect::<Result<FxHashMap<_, _>, _>>()?;
-        if let Some(byte) = (0..=u8::MAX).find(|byte| !ranks.contains_key(&[*byte][..])) {
-            return Err(Error::Ranks(format!("byte 0x{byte:02X} has no token")));
-        }
-        Ok(Self(ranks))
-    }
-
-    fn rank(&self, bytes: &[u8]) -> Rank {
-        self.0.get(bytes).copied().unwrap_or(NO_RANK)
-    }
-
-    /// Token count of one regex piece, as `encode_ordinary` would produce.
-    pub(super) fn count_piece(&self, piece: &[u8], scratch: &mut MergeScratch) -> usize {
-        if piece.len() < 2 || self.0.contains_key(piece) {
-            return 1;
-        }
-        scratch.reset(piece.len());
-        for start in 0..piece.len() - 1 {
-            scratch.set_rank(start, self.rank(&piece[start..start + 2]));
-        }
-        let mut parts = piece.len();
-        while let Some(Reverse((rank, start))) = scratch.heap.pop() {
-            if scratch.next[start] == END || scratch.rank[start] != rank {
-                continue;
-            }
-            let merged = scratch.next[start];
-            let after = scratch.next[merged];
-            scratch.next[merged] = END;
-            scratch.next[start] = after;
-            parts -= 1;
-            if after < piece.len() {
-                scratch.prev[after] = start;
-                scratch.set_rank(start, self.rank(&piece[start..scratch.end(after)]));
-            } else {
-                scratch.rank[start] = NO_RANK;
-            }
-            let before = scratch.prev[start];
-            if before != END {
-                scratch.set_rank(before, self.rank(&piece[before..scratch.end(start)]));
-            }
-        }
-        parts
+impl TokenCounter {
+    pub fn from_tiktoken(encoding: &str) -> Result<Self, Error> {
+        TiktokenTokenizer::from_name(encoding)
+            .map(Self::new)
+            .map_err(Error::from)
     }
 }
 
-fn parse_line(line: &str) -> Result<(Box<[u8]>, Rank), Error> {
-    let (token, rank) = line
-        .split_once(' ')
-        .ok_or_else(|| Error::Ranks(format!("line without a rank: {line:?}")))?;
-    let bytes = STANDARD
-        .decode(token)
-        .map_err(|error| Error::Ranks(format!("token is not base64: {error}")))?;
-    let rank = rank
-        .parse()
-        .map_err(|error| Error::Ranks(format!("rank is not an integer: {error}")))?;
-    Ok((bytes.into_boxed_slice(), rank))
-}
-
-/// Buffers reused across the pieces of one text. Parts are addressed by the
-/// byte offset they start at, which also gives the leftmost-pair tie break.
-#[derive(Default)]
-pub(super) struct MergeScratch {
-    next: Vec<usize>,
-    prev: Vec<usize>,
-    rank: Vec<Rank>,
-    heap: BinaryHeap<Reverse<(Rank, usize)>>,
-}
-
-impl MergeScratch {
-    fn reset(&mut self, len: usize) {
-        self.next.clear();
-        self.next.extend(1..=len);
-        self.prev.clear();
-        self.prev.push(END);
-        self.prev.extend(0..len - 1);
-        self.rank.clear();
-        self.rank.resize(len, NO_RANK);
-        self.heap.clear();
-    }
-
-    fn end(&self, start: usize) -> usize {
-        self.next[start]
-    }
-
-    fn set_rank(&mut self, start: usize, rank: Rank) {
-        self.rank[start] = rank;
-        if rank != NO_RANK {
-            self.heap.push(Reverse((rank, start)));
-        }
+impl Tokenizer for TiktokenTokenizer {
+    fn count_tokens(&self, text: &str) -> Result<usize, Error> {
+        Ok(TiktokenTokenizer::count_tokens(self, text))
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use rand::rngs::StdRng;
-    use rand::{Rng, SeedableRng};
-
-    use super::*;
-
-    fn ranks() -> MergeRanks {
-        let path = concat!(
-            env!("CARGO_MANIFEST_DIR"),
-            "/../../../litellm/litellm_core_utils/tokenizers/9b5ad71b2ce5302211f9c61530b329a4922fc6a4"
-        );
-        MergeRanks::parse(&std::fs::read_to_string(path).expect("cl100k rank file is in the repo"))
-            .expect("rank file parses")
-    }
-
-    /// tiktoken's `_byte_pair_merge`, transcribed, as the reference.
-    fn reference_count(ranks: &MergeRanks, piece: &[u8]) -> usize {
-        if piece.len() < 2 || ranks.0.contains_key(piece) {
-            return 1;
-        }
-        let mut parts: Vec<(usize, Rank)> = (0..piece.len() - 1)
-            .map(|index| (index, ranks.rank(&piece[index..index + 2])))
-            .chain([(piece.len() - 1, NO_RANK), (piece.len(), NO_RANK)])
-            .collect();
-        let get_rank = |parts: &[(usize, Rank)], index: usize| {
-            if index + 3 < parts.len() {
-                ranks.rank(&piece[parts[index].0..parts[index + 3].0])
-            } else {
-                NO_RANK
-            }
-        };
-        loop {
-            let Some(index) = parts[..parts.len() - 1]
-                .iter()
-                .enumerate()
-                .filter(|(_, (_, rank))| *rank != NO_RANK)
-                .min_by_key(|(index, (_, rank))| (*rank, *index))
-                .map(|(index, _)| index)
-            else {
-                return parts.len() - 1;
-            };
-            if index > 0 {
-                parts[index - 1].1 = get_rank(&parts, index - 1);
-            }
-            parts[index].1 = get_rank(&parts, index);
-            parts.remove(index + 1);
-        }
-    }
-
-    #[test]
-    fn every_byte_is_a_token() {
-        let ranks = ranks();
-        assert_eq!(ranks.0.len(), 100_256);
-        assert!((0..=u8::MAX).all(|byte| ranks.rank(&[byte]) != NO_RANK));
-    }
-
-    #[test]
-    fn heap_merge_matches_tiktokens_merge_loop() {
-        let ranks = ranks();
-        let mut scratch = MergeScratch::default();
-        let mut rng = StdRng::seed_from_u64(99);
-        let alphabet = b" abcdeorstn.,'\n\xc3\xa9\xe2\x82\xac0123";
-        for _ in 0..20_000 {
-            let piece: Vec<u8> = (0..rng.gen_range(1..24))
-                .map(|_| alphabet[rng.gen_range(0..alphabet.len())])
-                .collect();
-            assert_eq!(
-                ranks.count_piece(&piece, &mut scratch),
-                reference_count(&ranks, &piece),
-                "piece {:?}",
-                String::from_utf8_lossy(&piece)
-            );
-        }
-    }
-
-    #[test]
-    fn long_repeated_runs_cost_close_to_linear() {
-        let ranks = ranks();
-        let mut scratch = MergeScratch::default();
-        let mut time = |len: usize| {
-            let piece = vec![b' '; len];
-            let started = std::time::Instant::now();
-            assert!(ranks.count_piece(&piece, &mut scratch) > 0);
-            started.elapsed()
-        };
-        let small = (0..3).map(|_| time(1 << 14)).min().unwrap();
-        let large = time(1 << 18);
-        assert!(
-            large < small * 64,
-            "{small:?} for 2^14 bytes, {large:?} for 2^18"
-        );
-    }
-
-    #[test]
-    fn malformed_rank_files_are_rejected() {
-        assert!(MergeRanks::parse("IQ==").is_err());
-        assert!(MergeRanks::parse("IQ== x").is_err());
-        assert!(MergeRanks::parse("!!! 1").is_err());
-        assert!(MergeRanks::parse("IQ== 1").is_err());
+impl From<UnsupportedTokenizer> for Error {
+    fn from(error: UnsupportedTokenizer) -> Self {
+        Self::UnsupportedTokenizer(error.0)
     }
 }